kangaroo-and-rabbit/karideo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[DEV] add basic backend

Browse Source
This commit is contained in:
Edouard DUPIN 2019-12-21 21:52:26 +01:00
parent 08b103b06a
commit bdc2e7d176
19 changed files with 2215 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
back/Dockerfile Executable file
View File

@ -0,0 +1,26 @@
FROM python:alpine
RUN pip install --upgrade pip
RUN pip install sanic
RUN pip install sanic-cors
RUN pip install sanic-simple-swagger
RUN pip install python-dateutil
RUN pip install realog
RUN pip install python-magic
RUN pip install pymediainfo
EXPOSE 80
ADD src /application/
WORKDIR /application/
CMD ["python", "-u", "./app_video.py"]

1
back/data_base/bdd_group.json Normal file
View File

@ -0,0 +1 @@
[]

1
back/data_base/bdd_saison.json Normal file
View File

@ -0,0 +1 @@
[]

43
back/data_base/bdd_type.json Normal file
View File

@ -0,0 +1,43 @@
[
{
"id": 0,
"name": "Documentary",
"description": "Documentary (annimals, space, earth...)"
},{
"id": 1,
"name": "Movie",
"description": "Movie with real humans (film)"
},{
"id": 2,
"name": "Annimation",
"description": "Annimation movies (film)"
},{
"id": 3,
"name": "Short Films",
"description": "Small movies (less 2 minutes)"
},{
"id": 4,
"name": "tv show",
"description": "Tv show form old peoples"
}, {
"id": 5,
"name": "Anniation tv show",
"description": "Tv show form young peoples"
}, {
"id": 6,
"name": "Theater",
"description": "recorder theater pices"
}, {
"id": 7,
"name": "One man show",
"description": "Recorded stand up"
}, {
"id": 8,
"name": "Concert",
"description": "Recorded concert"
}, {
"id": 9,
"name": "Opera",
"description": "Recorded Opera"
}
]

1
back/data_base/bdd_video.json Normal file
View File

@ -0,0 +1 @@
[]

10
back/docker-compose.yaml Executable file
View File

@ -0,0 +1,10 @@
version: '3'
services:
REST_video_service:
build: .
restart: always
image: yui.heero/video_rest_api
container_name: video_rest_api
ports:
- 15080:80

56
back/readme.md Executable file
View File

@ -0,0 +1,56 @@
REST video API
==============
REST API for video streaming for personal web / application interface
Download the project
====================
simply download the application:
```
mkdir WORKSPACE & cd $_
git clone http://xxx/HeeroYui/rest_video.git restvideo
cd rest_video
```
**Note:** It is important to remove ```-``` and ```_``` becose some docker remove these element in the network name _(like ubuntu ...)_
**Note:** The networkname of a docker compose is ```thefoldername_default```
Run the application
===================
Start the application:
```
docker-compose up -d
```
Stop the application:
```
docker-compose down
```
Restart the application (on the fly):
```
docker-compose up -d --force-recreate --build
```
Run the application (debug)
===========================
before the first run:
```
cp -r data_base data
```
```
./src/app_video.py
```
or
```
SANIC_REST_PORT=15080 ./src/app_video.py
```

137
back/src/api/data.py Normal file
View File

@ -0,0 +1,137 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2019, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import time
import json
import os
import sys
import datetime
import time, threading
import realog.debug as debug
from aiofiles import os as async_os
from pymediainfo import MediaInfo
from sanic import Sanic
from sanic import response
from sanic import views
from sanic import Blueprint
from sanic.exceptions import ServerError
from sanic.response import file_stream
from sanic_simple_swagger import swagger_blueprint, openapi_blueprint
from sanic_simple_swagger import doc
import tools
import data_interface
import data_global_elements
import hashlib
import shutil
tmp_value = 0
#curl -F 'file=@Totally_Spies.mp4;type=application/octet-stream' -H 'transfer-encoding:chunked' 127.0.0.1:15080/data -X POST -O; echo ;
def add(_app, _name_api):
elem_blueprint = Blueprint(_name_api)
"""
@elem_blueprint.get('/' + _name_api, strict_slashes=True)
@doc.summary("Show saisons")
@doc.description("Display a listing of the resource.")
@doc.produces(content_type='application/json')
async def list(request):
return response.json(data_global_elements.get_interface(_name_api).gets())
"""
@elem_blueprint.post('/' + _name_api, strict_slashes=True, stream=True)
@doc.summary("send new file data")
@doc.description("Create a new data file (associated with his sha512.")
#@doc.consumes(DataModel, location='body')#, required=True)
@doc.response_success(status=201, description='If successful created')
async def create(_request):
debug.info("request streaming " + str(_request));
args_with_blank_values = _request.headers
debug.info("List arguments: " + str(args_with_blank_values));
async def streaming(_response):
debug.info("streaming " + str(_response));
total_size = 0
temporary_file = os.path.join(_app.config['REST_TMP_DATA'], str(tmp_value) + ".tmp")
if not os.path.exists(_app.config['REST_TMP_DATA']):
os.makedirs(_app.config['REST_TMP_DATA'])
if not os.path.exists(_app.config['REST_MEDIA_DATA']):
os.makedirs(_app.config['REST_MEDIA_DATA'])
file_stream = open(temporary_file,"wb")
sha1 = hashlib.sha512()
while True:
body = await _request.stream.read()
if body is None:
debug.warning("empty body");
break
total_size += len(body)
debug.verbose("body " + str(len(body)) + "/" + str(total_size))
file_stream.write(body)
sha1.update(body)
file_stream.close()
print("SHA512: " + str(sha1.hexdigest()))
destination_filename = os.path.join(_app.config['REST_MEDIA_DATA'], str(sha1.hexdigest()))
if os.path.isfile(destination_filename) == True:
answer_data = {
"size": total_size,
"sha512": str(sha1.hexdigest()),
'filename': _request.headers["filename"],
'mime-type': _request.headers["mime-type"],
"already_exist": True,
}
await _response.write(json.dumps(answer_data, sort_keys=True, indent=4))
return
# move the file
shutil.move(temporary_file, destination_filename)
# collect media info ...
media_info = MediaInfo.parse(destination_filename)
data_metafile = {
"sha512": str(sha1.hexdigest()),
"size": total_size,
'filename': _request.headers["filename"],
'mime-type': _request.headers["mime-type"],
'media-info': json.loads(media_info.to_json())
}
tools.file_write_data(destination_filename + ".meta", json.dumps(data_metafile, sort_keys=True, indent=4))
answer_data = {
"size": total_size,
"sha512": str(sha1.hexdigest()),
'filename': _request.headers["filename"],
'mime-type': _request.headers["mime-type"],
"already_exist": True,
}
await _response.write(json.dumps(answer_data, sort_keys=True, indent=4))
return response.stream(streaming, content_type='application/json')
@elem_blueprint.get('/' + _name_api + '/<id:string>', strict_slashes=True)
@doc.summary("Show resources")
@doc.description("Display a listing of the resource.")
@doc.produces(content_type='application/json')
async def retrive(request, id):
filename = os.path.join(_app.config['REST_MEDIA_DATA'], id)
if os.path.isfile(filename) == True:
file_stat = await async_os.stat(filename)
headers = {"Content-Length": str(file_stat.st_size)}
return await file_stream(
filename,
headers=headers,
chunked=False,
)
raise ServerError("No data found", status_code=404)
_app.blueprint(elem_blueprint)

129
back/src/api/group.py Normal file
View File

@ -0,0 +1,129 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2019, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import time
import json
import os
import sys
import datetime
import time, threading
import realog.debug as debug
from sanic import Sanic
from sanic import response
from sanic import views
from sanic import Blueprint
from sanic.exceptions import ServerError
from sanic_simple_swagger import swagger_blueprint, openapi_blueprint
from sanic_simple_swagger import doc
import tools
import data_interface
import data_global_elements
def add(_app, _name_api):
elem_blueprint = Blueprint(_name_api)
class DataModelBdd:
id = int
name = str
data_global_elements.get_interface(_name_api).set_data_model(DataModelBdd)
class DataModel:
name = str
@elem_blueprint.get('/' + _name_api, strict_slashes=True)
@doc.summary("Show resources")
@doc.description("Display a listing of the resource.")
@doc.produces(content_type='application/json')
async def list(request):
return response.json(data_global_elements.get_interface(_name_api).gets())
@elem_blueprint.post('/' + _name_api, strict_slashes=True)
@doc.summary("Create new resource")
@doc.description("Store a newly created resource in storage.")
@doc.consumes(DataModel, location='body')#, required=True)
@doc.response_success(status=201, description='If successful created')
async def create(request):
return response.json(data_global_elements.get_interface(_name_api).post(request.json))
@elem_blueprint.post('/' + _name_api + "/find", strict_slashes=True)
@doc.summary("Create new resource if the name does not already exist")
@doc.description("Store a newly created resource in storage.")
@doc.consumes(DataModel, location='body')#, required=True)
@doc.response_success(status=201, description='If successful created')
async def find_with_name(request):
api = data_global_elements.get_interface(_name_api)
for elem in api.bdd:
if elem["name"] == request.json["name"]:
return response.json({"id": elem["id"]})
raise ServerError("No data found", status_code=404)
@elem_blueprint.get('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Show resources")
@doc.description("Display a listing of the resource.")
@doc.produces(content_type='application/json')
async def retrive(request, id):
value = data_global_elements.get_interface(_name_api).get(id)
if value != None:
return response.json(value)
raise ServerError("No data found", status_code=404)
@elem_blueprint.get('/' + _name_api + '/<id:int>/video', strict_slashes=True)
@doc.summary("get videos list")
@doc.description("List all the videos availlable for this group.")
@doc.produces(content_type='application/json')
async def retrive_video(request, id):
value = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "group_id", id]], filter=["id"])
if value != None:
return response.json(value)
raise ServerError("No data found", status_code=404)
@elem_blueprint.get('/' + _name_api + '/<id:int>/video_no_saison', strict_slashes=True)
@doc.summary("get videos list who have no saison")
@doc.description("List all the videos availlable for this group tht does not depend on saison.")
@doc.produces(content_type='application/json')
async def retrive_video_no_saison(request, id):
value = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "group_id", id], ["==", "saison_id", None]], filter=["id"])
if value != None:
return response.json(value)
raise ServerError("No data found", status_code=404)
@elem_blueprint.get('/' + _name_api + '/<id:int>/saison', strict_slashes=True)
@doc.summary("get videos list who have no saison")
@doc.description("List all the videos availlable for this group tht does not depend on saison.")
@doc.produces(content_type='application/json')
async def retrive_saison(request, id):
value = data_global_elements.get_interface(data_global_elements.API_SAISON).gets_where(select=[["==", "group_id", id]], filter=["id"])
if value != None:
return response.json(value)
raise ServerError("No data found", status_code=404)
@elem_blueprint.put('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Update resource")
@doc.description("Update the specified resource in storage.")
@doc.response_success(status=201, description='If successful updated')
async def update(request, id):
ret = data_global_elements.get_interface(_name_api).put(id)
return response.json({})
@elem_blueprint.delete('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Remove resource")
@doc.description("Remove the specified resource from storage.")
@doc.response_success(status=201, description='If successful deleted')
async def delete(request, id):
ret = data_global_elements.get_interface(_name_api).delete(id)
if ret == True:
return response.json({})
raise ServerError("No data found", status_code=404)
_app.blueprint(elem_blueprint)

43
back/src/api/root.py Normal file
View File

@ -0,0 +1,43 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2019, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import time
import json
import os
import sys
import datetime
import time, threading
import realog.debug as debug
from sanic import Sanic
from sanic import response
from sanic import views
from sanic import Blueprint
from sanic.exceptions import ServerError
from sanic_simple_swagger import swagger_blueprint, openapi_blueprint
from sanic_simple_swagger import doc
import tools
import data_interface
import data_global_elements
def add(_app):
@_app.route("/")
@doc.description("get api system information")
async def test(request):
return response.json({
"api-type": "video-broker",
"api-version": _app.config['API_VERSION'],
"title": _app.config['API_TITLE'],
"description": _app.config['API_DESCRIPTION'],
"contact": _app.config['API_CONTACT_EMAIL'],
"licence": _app.config['API_LICENSE_NAME']
})

112
back/src/api/saison.py Normal file
View File

@ -0,0 +1,112 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2019, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import time
import json
import os
import sys
import datetime
import time, threading
import realog.debug as debug
from sanic import Sanic
from sanic import response
from sanic import views
from sanic import Blueprint
from sanic.exceptions import ServerError
from sanic_simple_swagger import swagger_blueprint, openapi_blueprint
from sanic_simple_swagger import doc
import tools
import data_interface
import data_global_elements
def add(_app, _name_api):
elem_blueprint = Blueprint(_name_api)
class DataModelBdd:
id = int
number = int
group_id = int
data_global_elements.get_interface(_name_api).set_data_model(DataModelBdd)
class DataModel:
number = int
group_id = int
@elem_blueprint.get('/' + _name_api, strict_slashes=True)
@doc.summary("Show saisons")
@doc.description("Display a listing of the resource.")
@doc.produces(content_type='application/json')
async def list(request):
return response.json(data_global_elements.get_interface(_name_api).gets())
@elem_blueprint.post('/' + _name_api, strict_slashes=True)
@doc.summary("Create new saison")
@doc.description("Create a new saison for a aspecific group id.")
@doc.consumes(DataModel, location='body')#, required=True)
@doc.response_success(status=201, description='If successful created')
async def create(request):
return response.json(data_global_elements.get_interface(_name_api).post(request.json))
@elem_blueprint.post('/' + _name_api + "/find", strict_slashes=True)
@doc.summary("find a season existance")
@doc.description("return the ID of the season table.")
@doc.consumes(DataModel, location='body')
@doc.response_success(status=201, description='If successful created')
async def find_with_name(request):
api = data_global_elements.get_interface(_name_api)
for elem in api.bdd:
if elem["group_id"] == request.json["group_id"] \
and elem["number"] == request.json["number"]:
return response.json({"id": elem["id"]})
raise ServerError("No data found", status_code=404)
@elem_blueprint.get('/' + _name_api + '/<id:int>/video', strict_slashes=True)
@doc.summary("Show videos")
@doc.description("List all the videos availlable for this group.")
@doc.produces(content_type='application/json')
async def retrive_video(request, id):
value = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "saison_id", id]], filter=["id"])
if value != None:
return response.json(value)
raise ServerError("No data found", status_code=404)
@elem_blueprint.get('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Show resources")
@doc.description("Display a listing of the resource.")
@doc.produces(content_type='application/json')
async def retrive(request, id):
value = data_global_elements.get_interface(_name_api).get(id)
if value != None:
return response.json(value)
raise ServerError("No data found", status_code=404)
@elem_blueprint.put('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Update resource")
@doc.description("Update the specified resource in storage.")
@doc.response_success(status=201, description='If successful updated')
async def update(request, id):
ret = data_global_elements.get_interface(_name_api).put(id)
return response.json({})
@elem_blueprint.delete('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Remove resource")
@doc.description("Remove the specified resource from storage.")
@doc.response_success(status=201, description='If successful deleted')
async def delete(request, id):
ret = data_global_elements.get_interface(_name_api).delete(id)
if ret == True:
return response.json({})
raise ServerError("No data found", status_code=404)
_app.blueprint(elem_blueprint)

122
back/src/api/type.py Normal file
View File

@ -0,0 +1,122 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2019, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import time
import json
import os
import sys
import datetime
import time, threading
import realog.debug as debug
from sanic import Sanic
from sanic import response
from sanic import views
from sanic import Blueprint
from sanic.exceptions import ServerError
from sanic_simple_swagger import swagger_blueprint, openapi_blueprint
from sanic_simple_swagger import doc
import tools
import data_interface
import data_global_elements
def add(_app, _name_api):
elem_blueprint = Blueprint(_name_api)
class DataModelBdd:
id = int
name = str
description = str
data_global_elements.get_interface(_name_api).set_data_model(DataModelBdd)
class DataModel:
name = str
description = str
@elem_blueprint.get('/' + _name_api, strict_slashes=True)
@doc.summary("Show resources")
@doc.description("Display a listing of the resource.")
@doc.produces(content_type='application/json')
async def list(request):
return response.json(data_global_elements.get_interface(_name_api).gets())
@elem_blueprint.post('/' + _name_api, strict_slashes=True)
@doc.summary("Create new resource")
@doc.description("Store a newly created resource in storage.")
@doc.consumes(DataModel, location='body')#, required=True)
@doc.response_success(status=201, description='If successful created')
async def create(request):
return response.json(data_global_elements.get_interface(_name_api).post(request.json))
@elem_blueprint.get('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Show resources")
@doc.description("Display a listing of the resource.")
@doc.produces(content_type='application/json')
async def retrive(request, id):
value = data_global_elements.get_interface(_name_api).get(id)
if value != None:
return response.json(value)
raise ServerError("No data found", status_code=404)
@elem_blueprint.put('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Update resource")
@doc.description("Update the specified resource in storage.")
@doc.response_success(status=201, description='If successful updated')
async def update(request, id):
ret = data_global_elements.get_interface(_name_api).put(id)
return response.json({})
@elem_blueprint.delete('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Remove resource")
@doc.description("Remove the specified resource from storage.")
@doc.response_success(status=201, description='If successful deleted')
async def delete(request, id):
ret = data_global_elements.get_interface(_name_api).delete(id)
if ret == True:
return response.json({})
raise ServerError("No data found", status_code=404)
@elem_blueprint.get('/' + _name_api + '/<id:int>/count', strict_slashes=True)
@doc.summary("Count resources in this cathegory")
@doc.description("count resources in this cathegory, in the whole tree.")
@doc.produces(content_type='application/json')
async def count_values(request, id):
count_value = data_global_elements.get_interface(data_global_elements.API_VIDEO).count(select=[["==", "type_id", id]])
return response.json({"count":count_value})
@elem_blueprint.get('/' + _name_api + '/<id:int>/video', strict_slashes=True)
@doc.summary("List the whole video ids")
@doc.description("List all video availlable with this type (list of ids).")
@doc.produces(content_type='application/json')
async def retrive_video(request, id):
list_values = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "type_id", id]], filter=["id"])
return response.json(list_values)
@elem_blueprint.get('/' + _name_api + '/<id:int>/video_no_group', strict_slashes=True)
@doc.summary("List the whole video ids")
@doc.description("List all video availlable with this type (list of ids).")
@doc.produces(content_type='application/json')
async def retrive_video_no_group(request, id):
list_values = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "type_id", id], ["==", "group_id", None]], filter=["id"])
return response.json(list_values)
@elem_blueprint.get('/' + _name_api + '/<id:int>/group', strict_slashes=True)
@doc.summary("List the whole video ids")
@doc.description("List all video availlable with this type (list of ids).")
@doc.produces(content_type='application/json')
async def retrive_group(request, id):
list_values = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "type_id", id], ["!=", "group_id", None]], filter=["group_id"])
return response.json(list_values)
_app.blueprint(elem_blueprint)

156
back/src/api/video.py Normal file
View File

@ -0,0 +1,156 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2019, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import time
import json
import os
import sys
import copy
import datetime
import time, threading
import realog.debug as debug
from sanic import Sanic
from sanic import response
from sanic import views
from sanic import Blueprint
from sanic.exceptions import ServerError
from sanic_simple_swagger import swagger_blueprint, openapi_blueprint
from sanic_simple_swagger import doc
import tools
import data_interface
import data_global_elements
def generate_name(_value):
group_name = ""
if "group_id" in _value.keys():
group_property = data_global_elements.get_interface(data_global_elements.API_GROUP).get(_value["group_id"])
if group_property != None:
group_name = group_property["name"]
saison_number = ""
if "saison_id" in _value.keys():
saison_property = data_global_elements.get_interface(data_global_elements.API_SAISON).get(_value["saison_id"])
if saison_property != None:
saison_number = str(saison_property["number"])
if len(saison_number) == 1:
saison_number = "0" + saison_number
out = ""
if group_name != "":
out += group_name + "-"
if saison_number != "":
out += "s" + saison_number + "-"
if "episode" in _value.keys() and _value["episode"] != None:
if _value["episode"] < 10:
out += "e00" + str(_value["episode"]) + "-"
elif _value["episode"] < 100:
out += "e0" + str(_value["episode"]) + "-"
else:
out += "e" + str(_value["episode"]) + "-"
out += _value["name"]
if "time" in _value.keys() and _value["time"] != None:
out += "(" + _value["name"] + ")"
return out
def add(_app, _name_api):
elem_blueprint = Blueprint(_name_api)
class DataModelBdd:
id = int
sha512 = str
type_id = int
saison_id = [int, type(None)]
episode = [int, type(None)]
group_id = [int, type(None)]
name = str
description = [str, type(None)]
# creating time
create_date = str
# number of second
time = [int, type(None)]
data_global_elements.get_interface(_name_api).set_data_model(DataModelBdd)
class DataModel:
type_id = int
saison_id = int
episode = int
group_id = int
name = str
description = str
# creating time
create_date = str
# number of second
time = int
@elem_blueprint.get('/' + _name_api, strict_slashes=True)
@doc.summary("Show saisons")
@doc.description("Display a listing of the resource.")
@doc.produces(content_type='application/json')
async def list(request):
return response.json(data_global_elements.get_interface(_name_api).gets())
@elem_blueprint.post('/' + _name_api, strict_slashes=True)
@doc.summary("Create new saison")
@doc.description("Create a new saison for a aspecific group id.")
@doc.consumes(DataModel, location='body')#, required=True)
@doc.response_success(status=201, description='If successful created')
async def create(request):
for type_key in ["sha512","type_id","name"]:
if type_key not in request.json.keys():
raise ServerError("Bad Request: Missing Key '" + type_key + "'", status_code=400)
for type_key in ["date"]:
if type_key in request.json.keys():
raise ServerError("Forbidden: Must not be set Key '" + type_key + "'", status_code=403)
for type_key in ["saison_id","episode","time","group_id","description"]:
if type_key not in request.json.keys():
request.json[type_key] = None
request.json["create_date"] = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
#Find if already exist
list_elem = data_global_elements.get_interface(_name_api).find(["group_id", "sha512"], request.json);
for elem in list_elem:
return response.json(elem)
return response.json(data_global_elements.get_interface(_name_api).post(request.json))
@elem_blueprint.get('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Show resources")
@doc.description("Display a listing of the resource.")
@doc.produces(content_type='application/json')
async def retrive(request, id):
value = data_global_elements.get_interface(_name_api).get(id)
if value != None:
generated_name = generate_name(value)
tmp = copy.deepcopy(value)
tmp["generated_name"] = generated_name
return response.json(tmp)
raise ServerError("No data found", status_code=404)
@elem_blueprint.put('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Update resource")
@doc.description("Update the specified resource in storage.")
@doc.response_success(status=201, description='If successful updated')
async def update(request, id):
ret = data_global_elements.get_interface(_name_api).put(id)
return response.json({})
@elem_blueprint.delete('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Remove resource")
@doc.description("Remove the specified resource from storage.")
@doc.response_success(status=201, description='If successful deleted')
async def delete(request, id):
ret = data_global_elements.get_interface(_name_api).delete(id)
if ret == True:
return response.json({})
raise ServerError("No data found", status_code=404)
_app.blueprint(elem_blueprint)

106
back/src/app_video.py Executable file
View File

@ -0,0 +1,106 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2019, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
#pip install flask --user
#pip install flask_restful --user
#pip install python-dateutil --user
#pip install sanic --user
#pip install sanic_simple_swagger --user
from sanic import Sanic
from sanic import response
from sanic import views
from sanic import Blueprint
from sanic.exceptions import ServerError
from sanic_simple_swagger import swagger_blueprint, openapi_blueprint
from sanic_simple_swagger import doc
from spf import SanicPluginsFramework
import dateutil.parser
import time
import json
import os
import sys
import datetime
import time, threading
import realog.debug as debug
debug.enable_color()
import tools
import data_interface
import data_global_elements
from sanic_cors.extension import cors
app = Sanic(__name__)
spf = SanicPluginsFramework(app)
spf.register_plugin(cors, automatic_options=True)
app.config['API_VERSION'] = '1.0.0'
app.config['API_TITLE'] = 'Rest personal video API'
app.config['API_DESCRIPTION'] = 'Simple API for the Video broker.'
app.config['API_CONTACT_EMAIL'] = "yui.heero@gmail.com"
app.config['API_LICENSE_NAME'] = 'MPL 2.0'
app.config['API_LICENSE_URL'] = 'https://www.mozilla.org/en-US/MPL/2.0/'
app.config['schemes'] = ['http', 'https']
if "REST_TMP_DATA" not in app.config.keys():
app.config['REST_TMP_DATA'] = "tmp"
if "REST_MEDIA_DATA" not in app.config.keys():
app.config['REST_MEDIA_DATA'] = os.path.join("data", "media")
if "REST_DATA" not in app.config.keys():
app.config['REST_DATA'] = "data"
if "REST_HOST" not in app.config.keys():
app.config['REST_HOST'] = "localhost"
if "REST_PORT" not in app.config.keys():
app.config['REST_PORT'] = "80"
app.blueprint(openapi_blueprint)
app.blueprint(swagger_blueprint)
def add_interface(_name):
data_global_elements.add_interface(_name, data_interface.DataInterface(_name, os.path.join(tools.get_run_path(), app.config['REST_DATA'], "bdd_" + _name + ".json")))
add_interface(data_global_elements.API_TYPE)
add_interface(data_global_elements.API_GROUP)
add_interface(data_global_elements.API_SAISON)
add_interface(data_global_elements.API_VIDEO)
import api.root as api_root
api_root.add(app)
import api.type as api_type
api_type.add(app, data_global_elements.API_TYPE)
import api.group as api_group
api_group.add(app, data_global_elements.API_GROUP)
import api.saison as api_saison
api_saison.add(app, data_global_elements.API_SAISON)
import api.video as api_video
api_video.add(app, data_global_elements.API_VIDEO)
import api.data as api_data
api_data.add(app, data_global_elements.API_DATA)
if __name__ == "__main__":
debug.info("Start REST application: " + str(app.config['REST_HOST']) + ":" + str(app.config['REST_PORT']))
app.config.REQUEST_MAX_SIZE=10*1024*1024*1024
app.run(host=app.config['REST_HOST'], port=int(app.config['REST_PORT']))
debug.info("END program");
sys.exit(0)

41
back/src/data_global_elements.py Normal file
View File

@ -0,0 +1,41 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2019, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
interfaces = {}
def get_list_interface():
global interfaces
return interfaces
def get_interface(_name):
global interfaces
return interfaces[_name]
def add_interface(_name, _interface):
global interfaces
interfaces[_name] = _interface
import time, threading
def check_save():
print(time.ctime())
for elem in interfaces.keys():
interfaces[elem].check_save()
threading.Timer(10, check_save).start()
check_save()
API_TYPE = "type"
API_GROUP = "group"
API_SAISON = "saison"
API_VIDEO = "video"
API_DATA = "data"

246
back/src/data_interface.py Normal file
View File

@ -0,0 +1,246 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2019, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import tools
import json
from realog import debug
import random
from sanic.exceptions import ServerError
##
## @breif Generic interface to access to the BDD (no BDD, direct file IO)
##
class DataInterface():
def __init__(self, _name, _file):
self.model = None
self.name = _name
self.file = _file
self.bdd = []
self.need_save = False
self.last_id = 0
if tools.exist(self.file) == False:
self.mark_to_store()
self.last_id = random.randint(20, 100)
else:
data = tools.file_read_data(self.file)
self.bdd = json.loads(data)
self.upgrade_global_bdd_id();
def set_data_model(self, _data_model):
self.model = _data_model
def check_with_model(self, _data):
if self.model == None:
return True
values = []
for elem in dir(self.model):
if elem[:2] == "__":
continue
values.append(elem)
have_error = False
for key in _data.keys():
if key not in values:
have_error = True
# TODO: ...
debug.warning("Add element that is not allowed " + key + " not in " + str(values))
for elem in values:
if key not in _data.keys():
have_error = True
# TODO: ...
debug.warning("Missing key " + elem + " not in " + str(_data.keys()))
if have_error == True:
return False
for key in _data.keys():
elem = getattr(self.model, key)
if type(elem) == list:
find_error = True
for my_type in elem:
if type(_data[key]) == my_type:
find_error = False
break
if find_error == True:
debug.warning("data : " + str(_data))
tmp_list = []
for my_type in elem:
tmp_list.append(my_type.__name__)
debug.warning("[key='" + key + "'] try to add wrong type in BDD " + type(_data[key]).__name__ + " is not: " + str(my_type))
else:
if type(_data[key]) != getattr(self.model, key):
debug.warning("data : " + str(_data))
debug.warning("[key='" + key + "'] try to add wrong type in BDD " + type(_data[key]).__name__ + " is not: " + getattr(self.model, key).__name__)
return False
return True
def upgrade_global_bdd_id(self):
for elem in self.bdd:
if 'id' not in elem.keys():
continue
if elem["id"] >= self.last_id:
self.last_id = elem["id"] + 1
def get_table_index(self, _id):
id_in_bdd = 0
for elem in self.bdd:
if 'id' in elem.keys() \
and elem["id"] == _id:
return id_in_bdd
id_in_bdd += 1
return None
##
## @brief Mark the current BDD to store all in File system (sync)
##
def mark_to_store(self):
self.need_save = True
##
## @brief Check if the Bdd need to be stored. It is stored if it has been requested.
## The BDD is store in a separate file and move in the old one. Safe way to store
##
def check_save(self):
if self.need_save == False:
return
debug.warning("Save bdd: " + self.file)
data = json.dumps(self.bdd, sort_keys=True, indent=4)
self.need_save = False
tools.file_write_data_safe(self.file, data)
def gets(self, filter=None):
debug.info("gets " + self.name)
if filter == None:
return self.bdd
return self.filter_object_values(self.bdd, filter)
def gets_where(self, select, filter=None):
debug.info("gets " + self.name)
tmp_list = self.get_sub_list(self.bdd, select)
return self.filter_object_values(tmp_list, filter);
def get(self, _id):
debug.info("get " + self.name + ": " + str(_id))
for elem in self.bdd:
if 'id' in elem.keys() \
and elem["id"] == _id:
return elem
return None
def delete(self, _id):
debug.info("delete " + self.name + ": " + str(_id))
id_in_bdd = self.get_table_index(_id)
if id_in_bdd == None:
return False
del self.bdd[id_in_bdd]
self.mark_to_store()
return True
def put(self, _id, _value):
debug.info("put " + self.name + ": " + str(_id))
id_in_bdd = self.get_table_index(_id)
if id_in_bdd == None:
return False
_value["id"] = _id
self.bdd[id_in_bdd] = _value
self.mark_to_store()
return True
def post(self, _value):
debug.info("post " + self.name)
_value["id"] = self.last_id
self.last_id += 1
if self.check_with_model(_value) == False:
raise ServerError("Corelation with BDD error", status_code=404)
self.bdd.append(_value)
self.mark_to_store()
return _value
# TODO : rework this
def find(self, _list_token, _values):
out = []
for elem in self.bdd:
find = True
for token in _list_token:
if elem[token] != _values[token]:
find = False
break
if find == True:
out.append(elem)
return out
def count(self, select = None):
if select == None:
return len(self.bdd)
tmp = self.get_sub_list(self.bdd, select)
return len(tmp)
def get_sub_list(self, _values, _select):
out = []
for elem in _values:
find = True
if len(_select) == 0:
find = False
for elem_select in _select:
if len(elem_select) != 3:
raise ServerError("Internal Server Error: wrong select definition", 500)
type_check = elem_select[0]
token = elem_select[1]
value = elem_select[2]
if token in elem.keys():
if type_check == "==":
if not (elem[token] == value):
find = False
break
elif type_check == "!=":
if not (elem[token] != value):
find = False
break
elif type_check == "<":
if not (elem[token] < value):
find = False
break
elif type_check == "<=":
if not (elem[token] <= value):
find = False
break
elif type_check == ">":
if not (elem[token] >= value):
find = False
break
elif type_check == ">=":
if not (elem[token] >= value):
find = False
break
else:
raise ServerError("Internal Server Error: unknow comparing type ...", 500)
else:
find = False
break
if find == True:
out.append(elem)
return out
def filter_object_values(self, _values, _filter):
out = []
if len(_filter) == 1:
token = _filter[0]
for elem in _values:
if token not in elem.keys():
continue
if elem[token] not in out:
out.append(elem[token])
return out
for elem in _values:
element_out = {}
for token in _filter:
if token not in elem.keys():
continue
element_out[token] = elem[token]
out.append(element_out)
return out

133
back/src/tools.py Normal file
View File

@ -0,0 +1,133 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2012, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import os
import shutil
import errno
import fnmatch
import stat
# Local import
from realog import debug
"""
"""
def get_run_path():
return os.getcwd()
"""
"""
def get_current_path(file):
return os.path.dirname(os.path.realpath(file))
def create_directory_of_file(file):
debug.info("Create directory of path: '" + file + "'")
path = os.path.dirname(file)
debug.info("Create directory: '" + path + "'")
try:
os.stat(path)
except:
os.makedirs(path)
def get_list_sub_path(path):
# TODO : os.listdir(path)
for dirname, dirnames, filenames in os.walk(path):
return dirnames
return []
def remove_path_and_sub_path(path):
if os.path.isdir(path):
debug.verbose("remove path : '" + path + "'")
shutil.rmtree(path)
def remove_file(path):
if os.path.isfile(path):
os.remove(path)
elif os.path.islink(path):
os.remove(path)
def exist(path):
if os.path.isdir(path):
return True
if os.path.isfile(path):
return True
if os.path.islink(path):
return True
return False
def file_size(path):
if not os.path.isfile(path):
return 0
statinfo = os.stat(path)
return statinfo.st_size
def file_read_data(path, binary=False):
debug.verbose("path= " + path)
if not os.path.isfile(path):
return ""
if binary == True:
file = open(path, "rb")
else:
file = open(path, "r")
data_file = file.read()
file.close()
return data_file
def version_to_string(version):
version_ID = ""
for id in version:
if len(version_ID) != 0:
if type(id) == str:
version_ID += "-"
else:
version_ID += "."
version_ID += str(id)
return version_ID
##
## @brief Write data in a specific path.
## @param[in] path Path of the data might be written.
## @param[in] data Data To write in the file.
## @param[in] only_if_new (default: False) Write data only if data is different.
## @return True Something has been copied
## @return False Nothing has been copied
##
def file_write_data(path, data, only_if_new=False):
if only_if_new == True:
if os.path.exists(path) == True:
old_data = file_read_data(path)
if old_data == data:
return False
#real write of data:
create_directory_of_file(path)
file = open(path, "w")
file.write(data)
file.close()
return True
def file_write_data_safe(path, data):
#real write of data:
create_directory_of_file(path)
file = open(path + ".tmp", "w")
file.write(data)
file.close()
shutil.move(path + ".tmp", path)
return True
def list_to_str(list):
if type(list) == type(str()):
return list + " "
else:
result = ""
# mulyiple imput in the list ...
for elem in list:
result += list_to_str(elem)
return result

42
back/tools/sendFile.py Executable file
View File

@ -0,0 +1,42 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2019, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import os
import sys
import requests # pip install requests
class upload_in_chunks(object):
def __init__(self, filename, chunksize=1 << 13):
self.filename = filename
self.chunksize = chunksize
self.totalsize = os.path.getsize(filename)
self.readsofar = 0
def __iter__(self):
with open(self.filename, 'rb') as file:
while True:
data = file.read(self.chunksize)
if not data:
sys.stderr.write("\n")
break
self.readsofar += len(data)
percent = self.readsofar * 1e2 / self.totalsize
sys.stderr.write("\rSendfing data: {percent:3.0f}% {size:14.0f} / {total_size}".format(percent=percent, size=self.readsofar, total_size=self.totalsize))
yield data
def __len__(self):
return self.totalsize
filename = 'Totally_Spies.mp4'
result = requests.post("http://127.0.0.1:15080/data", data=upload_in_chunks(filename, chunksize=4096))
print("result : " + str(result) + " " + result.text)#str(dir(result)))

810
back/tools/sendLocalData.py Executable file
View File

@ -0,0 +1,810 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2019, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import os
import copy
import sys
import hashlib
import requests # pip install requests
import realog.debug as debug
import magic
import json
debug.enable_color();
class upload_in_chunks(object):
def __init__(self, filename, chunksize=1 + 13):
self.filename = filename
self.chunksize = chunksize
self.totalsize = os.path.getsize(filename)
self.readsofar = 0
def __iter__(self):
with open(self.filename, 'rb') as file:
while True:
data = file.read(self.chunksize)
if not data:
sys.stderr.write("\n")
break
self.readsofar += len(data)
percent = self.readsofar * 1e2 / self.totalsize
sys.stderr.write("\rSendfing data: {percent:3.0f}% {size:14.0f} / {total_size}".format(percent=percent, size=self.readsofar, total_size=self.totalsize))
yield data
def __len__(self):
return self.totalsize
#filename = 'Totally_Spies.mp4'
#result = requests.post("http://127.0.0.1:15080/data", data=upload_in_chunks(filename, chunksize=4096))
#debug.info("result : " + str(result) + " " + result.text)#str(dir(result)))
def extract_and_remove(_input_value, _start_mark, _stop_mark):
values = []
out = ""
inside = False
inside_data = ""
for it in _input_value:
if inside == False \
and it == _start_mark:
inside = True
elif inside == True \
and it == _stop_mark:
inside = False
values.append(inside_data)
inside_data = ""
elif inside == True:
inside_data += it
else:
out += it
return (out, values)
def create_directory_of_file(_file):
path = os.path.dirname(_file)
try:
os.stat(path)
except:
os.makedirs(path)
##
## @brief Write data in a specific path.
## @param[in] path Path of the data might be written.
## @param[in] data Data To write in the file.
## @param[in] only_if_new (default: False) Write data only if data is different.
## @return True Something has been copied
## @return False Nothing has been copied
##
def file_write_data(_path, _data, _only_if_new=False):
if _only_if_new == True:
if os.path.exists(_path) == True:
old_data = file_read_data(_path)
if old_data == _data:
return False
#real write of data:
create_directory_of_file(_path)
file = open(_path, "w")
file.write(_data)
file.close()
return True
def get_modify_time(_path):
return os.stat(_path).st_mtime
def file_read_data(_path, _binary=False):
debug.verbose("path= " + _path)
if not os.path.isfile(_path):
return ""
if _binary == True:
file = open(_path, "rb")
else:
file = open(_path, "r")
data_file = file.read()
file.close()
return data_file
def calculate_sha512(_path):
sha1 = hashlib.sha512()
file = open(_path, "rb")
while True:
body = file.read(4096)
sha1.update(body)
file.close()
return str(sha1.hexdigest())
def push_video_file(_path, _basic_key={}):
file_name, file_extension = os.path.splitext(_path);
debug.info("Send file: '" + file_name + "' with extention " + file_extension)
# internal file_extension ...
if file_extension == "sha512":
debug.verbose("file: '" + _path + "' sha512 extention ...")
return True
debug.info("Add media : '" + _path + "'")
if file_extension[1:] not in ["avi", "mkv", "mov", "mp4", "ts"] \
and file_name not in ["cover_1.jpg","cover_1.png", "cover_1.till", "cover_1.bmp", "cover_1.tga"]:
debug.warning("Not send file : " + _path + " Not manage file_extension... " + file_extension)
return False
if file_name in ["cover_1.jpg","cover_1.png", "cover_1.till", "cover_1.bmp", "cover_1.tga"]:
# find a cover...
debug.warning("Not send cover Not managed ... : " + _path + " Not manage ...")
"""
debug.info("Send cover for: " + _basic_key["series-name"] + " " + _basic_key["saison"]);
if _basic_key["series-name"] == "":
debug.error(" ==> can not asociate at a specific seri");
return False;
etk::String groupName = _basic_key["series-name"];
if _basic_key["saison"] != "":
groupName += ":" + _basic_key["saison"];
auto sending = _srv.setGroupCover(zeus::File::create(_path.getString(), ""), groupName);
sending.onSignal(progressCallback);
sending.waitFor(echrono::seconds(20000));
"""
return True
"""
if etk::path::exist(_path + ".sha512") == True:
debug.verbose("file sha512 exist ==> read it");
uint64_t time_sha512 = get_modify_time(_path + ".sha512");
uint64_t time_elem = get_modify_time(_path);
storedSha512_file = file_read_data(_path + ".sha512")
debug.verbose("file sha == " + storedSha512_file);
if time_elem > time_sha512:
debug.verbose("file time > sha time ==> regenerate new one ...");
# check the current sha512
storedSha512 = calculate_sha512(_path);
debug.verbose("calculated new sha'" + storedSha512 + "'");
if storedSha512_file != storedSha512:
# need to remove the old sha file
auto idFileToRemove_fut = _srv.getId(storedSha512_file).waitFor(echrono::seconds(2));
if idFileToRemove_fut.hasError() == True:
debug.error("can not remove the remote file with sha " + storedSha512_file);
else:
debug.info("Remove old deprecated file: " + storedSha512_file);
_srv.remove(idFileToRemove_fut.get());
# note, no need to wait the call is async ... and the user does not interested with the result ...
# store new sha512 ==> this update tile too ...
file.open(etk::io::OpenMode::Write);
file.writeAll(storedSha512);
file.close();
else:
# store new sha512
/*
storedSha512 = file.readAllString();
file.open(etk::io::OpenMode::Read);
file.writeAll(storedSha512);
file.close();
*/
storedSha512 = storedSha512_file;
debug.verbose("read all sha from the file'" + storedSha512 + "'");
else:
"""
"""
if True:
storedSha512 = calculate_sha512(_path)
file_write_data(_path + ".sha512", storedSha512);
debug.info("calculate and store sha512 '" + storedSha512 + "'");
debug.info("check file existance: sha='" + storedSha512 + "'");
"""
# push only if the file exist
"""
# TODO : Check the metadata updating ...
auto idFile_fut = _srv.getId(storedSha512).waitFor(echrono::seconds(2));
if idFile_fut.hasError() == False:
# media already exit ==> stop here ...
return True;
# TODO: Do it better ==> add the calback to know the push progression ...
debug.verbose("Add File : " + _path + " sha='" + storedSha512 + "'");
auto sending = _srv.add(zeus::File::create(_path, storedSha512));
sending.onSignal(progressCallback);
debug.verbose("Add done ... now waiting ... ");
uint32_t mediaId = sending.waitFor(echrono::seconds(20000)).get();
debug.verbose("END WAITING ... ");
if mediaId == 0:
debug.error("Get media ID = 0 With no error");
return False;
"""
mime = magic.Magic(mime=True)
mime_type = mime.from_file(_path)
headers_values = {'filename': _path, 'mime-type': mime_type}
result_send_data = requests.post("http://127.0.0.1:15080/data", headers=headers_values, data=upload_in_chunks(_path, chunksize=4096))
debug.info("result *********** : " + str(result_send_data) + " " + result_send_data.text)
file_name = os.path.basename(file_name)
debug.info("Find file_name : '" + file_name + "'");
# Remove Date (XXXX) or other titreadsofarle
file_name, dates = extract_and_remove(file_name, '(', ')');
have_date = False
have_Title = False
for it in dates:
if len(it) == 0:
continue
if it[0] == '0' \
or it[0] == '1' \
or it[0] == '2' \
or it[0] == '3' \
or it[0] == '4' \
or it[0] == '5' \
or it[0] == '6' \
or it[0] == '7' \
or it[0] == '8' \
or it[0] == '9':
# find a date ...
if have_date == True:
debug.info(" '" + file_name + "'")
debug.error("Parse Date error : () : " + it + " ==> multiple date")
continue
have_date = True
_basic_key["date"] = it
else:
if have_Title == True:
debug.info(" '" + file_name + "'")
debug.error("Parse Title error : () : " + it + " ==> multiple title")
continue
have_Title = True
# Other title
_basic_key.set["title2"] = it;
# Remove the actors [XXX YYY][EEE TTT]...
file_name, actors = extract_and_remove(file_name, '[', ']');
if len(actors) > 0:
debug.info(" '" + file_name + "'")
actor_list = []
for it_actor in actors:
if actor_list != "":
actor_list += ";"
actor_list.append(it_actor)
_basic_key["actors"] = actor_list
list_element_base = file_name.split('-')
debug.warning("==> Title file: " + file_name)
debug.warning("==> Title cut : " + str(list_element_base))
list_element = [];
tmp_start_string = "";
iii = 0
while iii <len(list_element_base):
if list_element_base[iii][0] != 's' \
and list_element_base[iii][0] != 'e':
if tmp_start_string != "":
tmp_start_string += '-'
tmp_start_string += list_element_base[iii]
else:
list_element.append(tmp_start_string)
tmp_start_string = ""
while iii<len(list_element_base):
list_element.append(list_element_base[iii])
iii += 1
iii += 1
debug.warning("==> start elem: " + str(tmp_start_string))
if tmp_start_string != "":
list_element.append(tmp_start_string)
debug.warning("==> list_element : " + str(list_element))
if len(list_element) == 1:
# nothing to do , it might be a film ...
_basic_key["title"] = list_element[0]
else:
if len(list_element) > 3 \
and list_element[1][0] == 's' \
and list_element[2][0] == 'e':
debug.warning("Parse format: xxx-sXX-eXX-kjhlkjlkj(1234).*")
# internal formalisme ...
saison = -1;
episode = -1;
series_name = list_element[0];
_basic_key["series-name"] = series_name
full_episode_name = list_element[3]
for yyy in range(4, len(list_element)):
full_episode_name += "-" + list_element[yyy]
_basic_key["title"] = full_episode_name
if list_element[1][1:] == "XX":
# saison unknow ... ==> nothing to do ...
#saison = 123456789;
pass
else:
saison = int(list_element[1][1:]);
if list_element[2][1:] == "XX":
# episode unknow ... ==> nothing to do ...
pass
else:
episode = int(list_element[2][1:]);
_basic_key["episode"] = int(episode)
debug.info("Find a internal mode series: :");
debug.info(" origin : '" + file_name + "'");
saisonPrint = "XX";
episodePrint = "XX";
if saison < 0:
# nothing to do
pass
else:
saisonPrint = str(saison)
_basic_key["saison"] = saison
if episode < 0:
# nothing to do
pass
elif episode < 10:
episodePrint = "0" + str(episode);
_basic_key["episode"] = episode
else:
episodePrint = str(episode);
_basic_key["episode"] = episode
debug.info(" ==> '" + series_name + "-s" + saisonPrint + "-e" + episodePrint + "-" + full_episode_name + "'");
elif len(list_element) > 2 \
and list_element[1][0] == 'e':
debug.warning("Parse format: xxx-eXX-kjhlkjlkj(1234).*")
# internal formalisme ...
saison = -1;
episode = -1;
series_name = list_element[0];
_basic_key["series-name"] = series_name
full_episode_name = list_element[2]
for yyy in range(3, len(list_element)):
full_episode_name += "-" + list_element[yyy]
_basic_key["title"] = full_episode_name
if list_element[1][1:] == "XX":
# episode unknow ... ==> nothing to do ...
pass
else:
episode = int(list_element[1][1:]);
_basic_key["episode"] = int(episode)
debug.info("Find a internal mode series: :");
debug.info(" origin : '" + file_name + "'");
saisonPrint = "XX";
episodePrint = "XX";
if episode < 0:
# nothing to do
pass
elif episode < 10:
episodePrint = "0" + str(episode);
_basic_key["episode"] = episode
else:
episodePrint = str(episode);
_basic_key["episode"] = episode
debug.info(" ==> '" + series_name + "-s" + saisonPrint + "-e" + episodePrint + "-" + full_episode_name + "'");
result_send_data_json = json.loads(result_send_data.text)
debug.info("pared meta data: " + json.dumps(_basic_key, sort_keys=True, indent=4))
data_model = {
"type_id": _basic_key["type"],
"sha512": result_send_data_json["sha512"],
#"group_id": int,
"name": _basic_key["title"],
# number of second
"time": None,
}
for elem in ["date", "description", "episode"]: #["actors", "date", "description", "episode", "title2"]:
if elem in _basic_key.keys():
data_model[elem] = _basic_key[elem]
if "series-name" in _basic_key.keys():
result_group_data = requests.post("http://127.0.0.1:15080/group/find", data=json.dumps({"name":_basic_key["series-name"]}, sort_keys=True, indent=4))
debug.info("Create group ??? *********** : " + str(result_group_data) + " " + result_group_data.text)
if result_group_data.status_code == 404:
result_group_data = requests.post("http://127.0.0.1:15080/group", data=json.dumps({"name":_basic_key["series-name"]}, sort_keys=True, indent=4))
debug.info("yes we create new group *********** : " + str(result_group_data) + " " + result_group_data.text)
group_id = result_group_data.json()["id"]
data_model["group_id"] = group_id
if "saison" in _basic_key.keys():
result_saison_data = requests.post("http://127.0.0.1:15080/saison/find", data=json.dumps({"number":_basic_key["saison"], "group_id":group_id}, sort_keys=True, indent=4))
debug.info("Create saison ??? *********** : " + str(result_saison_data) + " " + result_saison_data.text)
if result_saison_data.status_code == 404:
result_saison_data = requests.post("http://127.0.0.1:15080/saison", data=json.dumps({"number":_basic_key["saison"], "group_id":group_id}, sort_keys=True, indent=4))
debug.info("yes we create new saison *********** : " + str(result_saison_data) + " " + result_saison_data.text)
saison_id = result_saison_data.json()["id"]
data_model["saison_id"] = saison_id
result_send_data = requests.post("http://127.0.0.1:15080/video", data=json.dumps(data_model, sort_keys=True, indent=4))
debug.info("result *********** : " + str(result_send_data) + " " + result_send_data.text)
return True
def install_video_path( _path, _basic_key = {}):
debug.info("Parse : '" + _path + "'");
list_sub_path = [fff for fff in os.listdir(_path) if os.path.isdir(os.path.join(_path, fff))]
for it_path in list_sub_path:
basic_key_tmp = copy.deepcopy(_basic_key)
debug.info("Add Sub path: '" + it_path + "'");
if len(basic_key_tmp) == 0:
debug.info("find A '" + it_path + "' " + str(len(basic_key_tmp)));
if it_path == "documentary":
basic_key_tmp["type"] = 0
elif it_path == "film":
basic_key_tmp["type"] = 1
elif it_path == "film-annimation":
basic_key_tmp["type"] = 2
elif it_path == "film-short":
basic_key_tmp["type"] = 3
elif it_path == "tv-show":
basic_key_tmp["type"] = 4
elif it_path == "tv-show-annimation":
basic_key_tmp["type"] = 5
elif it_path == "theater":
basic_key_tmp["type"] = 6
elif it_path == "one-man":
basic_key_tmp["type"] = 7
elif it_path == "concert":
basic_key_tmp["type"] = 8
elif it_path == "opera":
basic_key_tmp["type"] = 9
else:
debug.info("find B '" + it_path + "' " + str(len(basic_key_tmp)))
if it_path == "saison_01":
basic_key_tmp["saison"] = 1
elif it_path == "saison_02":
basic_key_tmp["saison"] = 2
elif it_path == "saison_03":
basic_key_tmp["saison"] = 3
elif it_path == "saison_04":
basic_key_tmp["saison"] = 4
elif it_path == "saison_05":
basic_key_tmp["saison"] = 5
elif it_path == "saison_06":
basic_key_tmp["saison"] = 6
elif it_path == "saison_07":
basic_key_tmp["saison"] = 7
elif it_path == "saison_08":
basic_key_tmp["saison"] = 8
elif it_path == "saison_09":
basic_key_tmp["saison"] = 9
elif it_path == "saison_10":
basic_key_tmp["saison"] = 10
elif it_path == "saison_11":
basic_key_tmp["saison"] = 11
elif it_path == "saison_12":
basic_key_tmp["saison"] = 12
elif it_path == "saison_13":
basic_key_tmp["saison"] = 13
elif it_path == "saison_14":
basic_key_tmp["saison"] = 14
elif it_path == "saison_15":
basic_key_tmp["saison"] = 15
elif it_path == "saison_16":
basic_key_tmp["saison"] = 16
elif it_path == "saison_17":
basic_key_tmp["saison"] = 17
elif it_path == "saison_18":
basic_key_tmp["saison"] = 18
elif it_path == "saison_19":
basic_key_tmp["saison"] = 19
elif it_path == "saison_20":
basic_key_tmp["saison"] = 20
elif it_path == "saison_21":
basic_key_tmp["saison"] = 21
elif it_path == "saison_22":
basic_key_tmp["saison"] = 22
elif it_path == "saison_23":
basic_key_tmp["saison"] = 23
elif it_path == "saison_24":
basic_key_tmp["saison"] = 24
elif it_path == "saison_25":
basic_key_tmp["saison"] = 25
elif it_path == "saison_26":
basic_key_tmp["saison"] = 26
elif it_path == "saison_27":
basic_key_tmp["saison"] = 27
elif it_path == "saison_28":
basic_key_tmp["saison"] = 28
elif it_path == "saison_29":
basic_key_tmp["saison"] = 29
else:
basic_key_tmp["series-name"] = it_path
debug.info("add a path " + os.path.join(_path, it_path) + " with keys " + str(basic_key_tmp))
install_video_path(os.path.join(_path, it_path), basic_key_tmp);
# Add files :
list_sub_file = [fff for fff in os.listdir(_path) if os.path.isfile(os.path.join(_path, fff))]
for it_file in list_sub_file:
basic_key_tmp = copy.deepcopy(_basic_key)
push_video_file(os.path.join(_path, it_file), basic_key_tmp);
property = {
"hostname": "127.0.0.1",
"port": 15080,
"login": None,
"password": None,
}
import death.Arguments as arguments
import death.ArgElement as arg_element
my_args = arguments.Arguments()
my_args.add_section("option", "Can be set one time in all case")
my_args.add("h", "help", desc="Display this help")
my_args.add("", "version", desc="Display the application version")
my_args.add("v", "verbose", list=[
["0","None"],
["1","error"],
["2","warning"],
["3","info"],
["4","debug"],
["5","verbose"],
["6","extreme_verbose"],
], desc="display debug level (verbose) default =2")
my_args.add("a", "action", list=[
["tree","List all the files in a tree view ..."],
["list","List all the files"],
["push","push a single file"],
["push_path","push a full folder"],
], desc="possible action")
my_args.add("c", "color", desc="Display message in color")
my_args.add("f", "folder", haveParam=False, desc="Display the folder instead of the git repository name")
local_argument = my_args.parse()
##
## @brief Display the help of this package.
##
def usage():
color = debug.get_color_set()
# generic argument displayed :
my_args.display()
exit(0)
##
## @brief Display the version of this package.
##
def version():
color = debug.get_color_set()
import pkg_resources
debug.info("version: 0.0.0")
foldername = os.path.dirname(__file__)
debug.info("source folder is: " + foldername)
exit(0)
folder = "dataPush"
requestAction = "list"
# preparse the argument to get the verbose element for debug mode
def parse_arg(argument):
debug.warning("parse arg : " + argument.get_option_name() + " " + argument.get_arg())
if argument.get_option_name() == "help":
usage()
return True
elif argument.get_option_name() == "version":
version()
return True
elif argument.get_option_name() == "verbose":
debug.set_level(int(argument.get_arg()))
return True
elif argument.get_option_name() == "color":
if check_boolean(argument.get_arg()) == True:
debug.enable_color()
else:
debug.disable_color()
return True
elif argument.get_option_name() == "folder":
folder = argument.get_arg()
return True
elif argument.get_option_name() == "action":
global requestAction
requestAction = argument.get_arg()
return True
return False
# parse default unique argument:
for argument in local_argument:
parse_arg(argument)
debug.info("==================================");
debug.info("== ZEUS test client start ==");
debug.info("==================================");
def show_video(elem_video_id, indent):
indent_data = ""
while indent > 0:
indent_data += "\t"
indent -= 1
result_video = requests.get("http://127.0.0.1:15080/video/" + str(elem_video_id) + "")
if result_video.status_code == 200:
video = result_video.json()
debug.info(indent_data + "- " + str(video["generated_name"]))
else:
debug.warning(indent_data + "get video id: " + str(elem_video_id) + " !!!!!! " + str(result_video.status_code) + "")
# ****************************************************************************************
# ** Clear All the data base ...
# ****************************************************************************************
if requestAction == "clear":
debug.info("============================================");
debug.info("== Clear data base: ");
debug.info("============================================");
# TODO : Do it :
debug.error("NEED to add check in cmd line to execute it ...");
"""
uint32_t count = remoteServiceVideo.count().wait().get();
debug.debug("have " + count + " medias");
for (uint32_t iii=0; iii<count ; iii += 1024:
uint32_t tmpMax = etk::min(iii + 1024, count);
debug.debug("read section " + iii + " -> " + tmpMax);
etk::Vector<uint32_t> list = remoteServiceVideo.getIds(iii,tmpMax).wait().get();
zeus::FutureGroup groupWait;
for (auto& it : list:
debug.info("remove ELEMENT : " + it);
groupWait.add(remoteServiceVideo.remove(it));
groupWait.waitFor(echrono::seconds(2000));
"""
debug.info("============================================");
debug.info("== DONE ==");
debug.info("============================================");
elif requestAction == "list":
debug.info("============================================");
debug.info("== list files: ");
debug.info("============================================");
list_types = requests.get("http://127.0.0.1:15080/type")
if list_types.status_code != 200:
debug.warning(" !! ca, ot get type list ... " + str(list_types.status_code) + "")
for elem in list_types.json():
debug.info(" get type id: " + str(elem["id"]))
debug.info(" name: " + str(elem["name"]))
# get the count of video in this type
result_count = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/count")
if result_count.status_code == 200:
debug.info(" count: " + str(result_count.json()["count"]))
else:
debug.warning(" count: !!!!!! " + str(result_count.status_code) + "")
# get all the video list
result_video = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/video")
if result_video.status_code == 200:
if len(result_video.json()) != 0:
debug.info(" List video: " + str(result_video.json()))
else:
debug.warning(" List video: !!!!!! " + str(result_video.status_code) + "")
# get list of groups for this type
result_groups = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/group")
if result_groups.status_code == 200:
if len(result_groups.json()) != 0:
debug.info(" List group: " + str(result_groups.json()))
else:
debug.warning(" List group: !!!!!! " + str(result_groups.status_code) + "")
# get list of video without groups
result_video_solo = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/video_no_group")
if result_video_solo.status_code == 200:
if len(result_video_solo.json()) != 0:
debug.info(" List video solo: " + str(result_video_solo.json()))
else:
debug.warning(" List video solo: !!!!!! " + str(result_video_solo.status_code) + "")
elif requestAction == "tree":
debug.info("============================================");
debug.info("== tree files: ");
debug.info("============================================");
list_types = requests.get("http://127.0.0.1:15080/type")
if list_types.status_code != 200:
debug.warning(" !! ca, ot get type list ... " + str(list_types.status_code) + "")
for elem in list_types.json():
debug.info("-------------------------------------------------")
debug.info(" " + str(elem["name"]))
debug.info("-------------------------------------------------")
# First get all the groups:
result_groups = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/group")
if result_groups.status_code == 200:
for elem_group_id in result_groups.json():
result_group = requests.get("http://127.0.0.1:15080/group/" + str(elem_group_id) + "")
if result_group.status_code == 200:
group = result_group.json()
debug.info("\to- " + str(group["name"]))
# step 1: all the saison:
result_saison_in_group = requests.get("http://127.0.0.1:15080/group/" + str(elem_group_id) + "/saison")
if result_saison_in_group.status_code == 200:
for elem_saison_id in result_saison_in_group.json():
result_saison = requests.get("http://127.0.0.1:15080/saison/" + str(elem_saison_id) + "")
if result_saison.status_code == 200:
debug.info("\t\t* saison " + str(result_saison.json()["number"]))
result_videos_in_saison = requests.get("http://127.0.0.1:15080/saison/" + str(result_saison.json()["id"]) + "/video")
if result_videos_in_saison.status_code == 200:
for elem_video_id in result_videos_in_saison.json():
show_video(elem_video_id, 3)
else:
debug.warning("\t\tget video in saison id: " + str(elem_saison_id) + " !!!!!! " + str(result_videos_in_saison.status_code) + "")
show_video(elem_video_id, 2)
else:
debug.warning("\t\tget saison id: " + str(elem_saison_id) + " !!!!!! " + str(result_saison.status_code) + "")
else:
debug.warning("\t\tget saison in group id: " + str(elem_group_id) + " !!!!!! " + str(result_saison_in_group.status_code) + "")
# step 2: all the video with no saison:
result_videos_in_group = requests.get("http://127.0.0.1:15080/group/" + str(elem_group_id) + "/video_no_saison")
if result_videos_in_group.status_code == 200:
for elem_video_id in result_videos_in_group.json():
show_video(elem_video_id, 2)
else:
debug.warning("\t\tget video in group id: " + str(elem_group_id) + " !!!!!! " + str(result_videos_in_group.status_code) + "")
else:
debug.warning("\tget group id: " + str(elem_group_id) + " !!!!!! " + str(result_group.status_code) + "")
else:
debug.warning("\t\tList group: !!!!!! " + str(result_groups.status_code) + "")
# get list of video without groups
result_video_solo = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/video_no_group")
if result_video_solo.status_code == 200:
for elem_video_id in result_video_solo.json():
show_video(elem_video_id, 1)
else:
debug.warning("\t\tList video solo: !!!!!! " + str(result_video_solo.status_code) + "")
"""
uint32_t count = remoteServiceVideo.count().wait().get();
debug.debug("have " + count + " medias");
for (uint32_t iii=0; iii<count ; iii += 1024:
uint32_t tmpMax = etk::min(iii + 1024, count);
debug.debug("read section " + iii + " -> " + tmpMax);
etk::Vector<uint32_t> list = remoteServiceVideo.getIds(iii, tmpMax).wait().get();
for (auto& it : list:
# Get the media
zeus::ProxyMedia media = remoteServiceVideo.get(it).waitFor(echrono::seconds(2000)).get();
if media.exist() == False:
debug.error("get media error");
return -1;
debug.debug(" Get title ...");
etk::String name = media.getMetadata("title").wait().get();
debug.debug(" Get series-name ...");
etk::String serie = media.getMetadata("series-name").wait().get();
debug.debug(" Get episode ...");
etk::String episode = media.getMetadata("episode").wait().get();
debug.debug(" Get saison ...");
etk::String saison = media.getMetadata("saison").wait().get();
etk::String outputDesc = "";
if serie != "":
outputDesc += serie + "-";
if saison != "":
outputDesc += "s" + saison + "-";
if episode != "":
outputDesc += "e" + episode + "-";
outputDesc += name;
debug.info("[" + it + "] '" + outputDesc + "'");
"""
debug.info("============================================");
debug.info("== DONE ==");
debug.info("============================================");
elif requestAction == "push":
debug.info("============================================");
debug.info("== push file: ");
debug.info("============================================");
push_video_file(folder);
debug.info("============================================");
debug.info("== DONE ==");
debug.info("============================================");
elif requestAction == "push_path":
debug.info("============================================");
debug.info("== push path: ");
debug.info("============================================");
install_video_path(folder);
debug.info("============================================");
debug.info("== DONE ==");
debug.info("============================================");
else:
debug.info("============================================");
debug.error("== Unknow action: '" + requestAction + "'");
debug.info("============================================");