From bdc2e7d176ecc926c51d6f1b67ca0f337c192fc0 Mon Sep 17 00:00:00 2001 From: Edouard DUPIN Date: Sat, 21 Dec 2019 21:52:26 +0100 Subject: [PATCH] [DEV] add basic backend --- back/Dockerfile | 26 + back/data_base/bdd_group.json | 1 + back/data_base/bdd_saison.json | 1 + back/data_base/bdd_type.json | 43 ++ back/data_base/bdd_video.json | 1 + back/docker-compose.yaml | 10 + back/readme.md | 56 +++ back/src/api/data.py | 137 ++++++ back/src/api/group.py | 129 +++++ back/src/api/root.py | 43 ++ back/src/api/saison.py | 112 +++++ back/src/api/type.py | 122 +++++ back/src/api/video.py | 156 ++++++ back/src/app_video.py | 106 ++++ back/src/data_global_elements.py | 41 ++ back/src/data_interface.py | 246 ++++++++++ back/src/tools.py | 133 +++++ back/tools/sendFile.py | 42 ++ back/tools/sendLocalData.py | 810 +++++++++++++++++++++++++++++++ 19 files changed, 2215 insertions(+) create mode 100755 back/Dockerfile create mode 100644 back/data_base/bdd_group.json create mode 100644 back/data_base/bdd_saison.json create mode 100644 back/data_base/bdd_type.json create mode 100644 back/data_base/bdd_video.json create mode 100755 back/docker-compose.yaml create mode 100755 back/readme.md create mode 100644 back/src/api/data.py create mode 100644 back/src/api/group.py create mode 100644 back/src/api/root.py create mode 100644 back/src/api/saison.py create mode 100644 back/src/api/type.py create mode 100644 back/src/api/video.py create mode 100755 back/src/app_video.py create mode 100644 back/src/data_global_elements.py create mode 100644 back/src/data_interface.py create mode 100644 back/src/tools.py create mode 100755 back/tools/sendFile.py create mode 100755 back/tools/sendLocalData.py diff --git a/back/Dockerfile b/back/Dockerfile new file mode 100755 index 0000000..5b06467 --- /dev/null +++ b/back/Dockerfile @@ -0,0 +1,26 @@ +FROM python:alpine + +RUN pip install --upgrade pip + +RUN pip install sanic + +RUN pip install sanic-cors + +RUN pip install sanic-simple-swagger + +RUN pip install python-dateutil + +RUN pip install realog + +RUN pip install python-magic + +RUN pip install pymediainfo + +EXPOSE 80 + +ADD src /application/ +WORKDIR /application/ +CMD ["python", "-u", "./app_video.py"] + + + diff --git a/back/data_base/bdd_group.json b/back/data_base/bdd_group.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/back/data_base/bdd_group.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/back/data_base/bdd_saison.json b/back/data_base/bdd_saison.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/back/data_base/bdd_saison.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/back/data_base/bdd_type.json b/back/data_base/bdd_type.json new file mode 100644 index 0000000..0245c90 --- /dev/null +++ b/back/data_base/bdd_type.json @@ -0,0 +1,43 @@ +[ + { + "id": 0, + "name": "Documentary", + "description": "Documentary (annimals, space, earth...)" + },{ + "id": 1, + "name": "Movie", + "description": "Movie with real humans (film)" + },{ + "id": 2, + "name": "Annimation", + "description": "Annimation movies (film)" + },{ + "id": 3, + "name": "Short Films", + "description": "Small movies (less 2 minutes)" + },{ + "id": 4, + "name": "tv show", + "description": "Tv show form old peoples" + }, { + "id": 5, + "name": "Anniation tv show", + "description": "Tv show form young peoples" + }, { + "id": 6, + "name": "Theater", + "description": "recorder theater pices" + }, { + "id": 7, + "name": "One man show", + "description": "Recorded stand up" + }, { + "id": 8, + "name": "Concert", + "description": "Recorded concert" + }, { + "id": 9, + "name": "Opera", + "description": "Recorded Opera" + } +] \ No newline at end of file diff --git a/back/data_base/bdd_video.json b/back/data_base/bdd_video.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/back/data_base/bdd_video.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/back/docker-compose.yaml b/back/docker-compose.yaml new file mode 100755 index 0000000..78fedb1 --- /dev/null +++ b/back/docker-compose.yaml @@ -0,0 +1,10 @@ +version: '3' +services: + REST_video_service: + build: . + restart: always + image: yui.heero/video_rest_api + container_name: video_rest_api + ports: + - 15080:80 + diff --git a/back/readme.md b/back/readme.md new file mode 100755 index 0000000..311448f --- /dev/null +++ b/back/readme.md @@ -0,0 +1,56 @@ +REST video API +============== + +REST API for video streaming for personal web / application interface + + +Download the project +==================== + +simply download the application: +``` +mkdir WORKSPACE & cd $_ +git clone http://xxx/HeeroYui/rest_video.git restvideo +cd rest_video +``` + +**Note:** It is important to remove ```-``` and ```_``` becose some docker remove these element in the network name _(like ubuntu ...)_ +**Note:** The networkname of a docker compose is ```thefoldername_default``` + + +Run the application +=================== + +Start the application: +``` +docker-compose up -d +``` + +Stop the application: +``` +docker-compose down +``` + +Restart the application (on the fly): +``` +docker-compose up -d --force-recreate --build +``` + + + +Run the application (debug) +=========================== +before the first run: +``` +cp -r data_base data +``` + +``` +./src/app_video.py +``` + +or +``` +SANIC_REST_PORT=15080 ./src/app_video.py +``` + diff --git a/back/src/api/data.py b/back/src/api/data.py new file mode 100644 index 0000000..9ae41ea --- /dev/null +++ b/back/src/api/data.py @@ -0,0 +1,137 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2019, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## + +import time +import json +import os +import sys +import datetime +import time, threading +import realog.debug as debug + +from aiofiles import os as async_os + +from pymediainfo import MediaInfo + +from sanic import Sanic +from sanic import response +from sanic import views +from sanic import Blueprint +from sanic.exceptions import ServerError +from sanic.response import file_stream + +from sanic_simple_swagger import swagger_blueprint, openapi_blueprint +from sanic_simple_swagger import doc + +import tools +import data_interface +import data_global_elements + +import hashlib +import shutil + +tmp_value = 0 + +#curl -F 'file=@Totally_Spies.mp4;type=application/octet-stream' -H 'transfer-encoding:chunked' 127.0.0.1:15080/data -X POST -O; echo ; + +def add(_app, _name_api): + elem_blueprint = Blueprint(_name_api) + """ + @elem_blueprint.get('/' + _name_api, strict_slashes=True) + @doc.summary("Show saisons") + @doc.description("Display a listing of the resource.") + @doc.produces(content_type='application/json') + async def list(request): + return response.json(data_global_elements.get_interface(_name_api).gets()) + """ + + @elem_blueprint.post('/' + _name_api, strict_slashes=True, stream=True) + @doc.summary("send new file data") + @doc.description("Create a new data file (associated with his sha512.") + #@doc.consumes(DataModel, location='body')#, required=True) + @doc.response_success(status=201, description='If successful created') + async def create(_request): + debug.info("request streaming " + str(_request)); + args_with_blank_values = _request.headers + debug.info("List arguments: " + str(args_with_blank_values)); + async def streaming(_response): + debug.info("streaming " + str(_response)); + total_size = 0 + temporary_file = os.path.join(_app.config['REST_TMP_DATA'], str(tmp_value) + ".tmp") + if not os.path.exists(_app.config['REST_TMP_DATA']): + os.makedirs(_app.config['REST_TMP_DATA']) + if not os.path.exists(_app.config['REST_MEDIA_DATA']): + os.makedirs(_app.config['REST_MEDIA_DATA']) + file_stream = open(temporary_file,"wb") + sha1 = hashlib.sha512() + while True: + body = await _request.stream.read() + if body is None: + debug.warning("empty body"); + break + total_size += len(body) + debug.verbose("body " + str(len(body)) + "/" + str(total_size)) + file_stream.write(body) + sha1.update(body) + file_stream.close() + print("SHA512: " + str(sha1.hexdigest())) + destination_filename = os.path.join(_app.config['REST_MEDIA_DATA'], str(sha1.hexdigest())) + if os.path.isfile(destination_filename) == True: + answer_data = { + "size": total_size, + "sha512": str(sha1.hexdigest()), + 'filename': _request.headers["filename"], + 'mime-type': _request.headers["mime-type"], + "already_exist": True, + } + await _response.write(json.dumps(answer_data, sort_keys=True, indent=4)) + return + # move the file + shutil.move(temporary_file, destination_filename) + + # collect media info ... + media_info = MediaInfo.parse(destination_filename) + data_metafile = { + "sha512": str(sha1.hexdigest()), + "size": total_size, + 'filename': _request.headers["filename"], + 'mime-type': _request.headers["mime-type"], + 'media-info': json.loads(media_info.to_json()) + } + tools.file_write_data(destination_filename + ".meta", json.dumps(data_metafile, sort_keys=True, indent=4)) + answer_data = { + "size": total_size, + "sha512": str(sha1.hexdigest()), + 'filename': _request.headers["filename"], + 'mime-type': _request.headers["mime-type"], + "already_exist": True, + } + await _response.write(json.dumps(answer_data, sort_keys=True, indent=4)) + return response.stream(streaming, content_type='application/json') + + @elem_blueprint.get('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Show resources") + @doc.description("Display a listing of the resource.") + @doc.produces(content_type='application/json') + async def retrive(request, id): + filename = os.path.join(_app.config['REST_MEDIA_DATA'], id) + if os.path.isfile(filename) == True: + file_stat = await async_os.stat(filename) + headers = {"Content-Length": str(file_stat.st_size)} + return await file_stream( + filename, + headers=headers, + chunked=False, + ) + raise ServerError("No data found", status_code=404) + + _app.blueprint(elem_blueprint) + + diff --git a/back/src/api/group.py b/back/src/api/group.py new file mode 100644 index 0000000..2f06a86 --- /dev/null +++ b/back/src/api/group.py @@ -0,0 +1,129 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2019, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## + +import time +import json +import os +import sys +import datetime +import time, threading +import realog.debug as debug + +from sanic import Sanic +from sanic import response +from sanic import views +from sanic import Blueprint +from sanic.exceptions import ServerError + +from sanic_simple_swagger import swagger_blueprint, openapi_blueprint +from sanic_simple_swagger import doc + +import tools +import data_interface +import data_global_elements + +def add(_app, _name_api): + elem_blueprint = Blueprint(_name_api) + + class DataModelBdd: + id = int + name = str + + data_global_elements.get_interface(_name_api).set_data_model(DataModelBdd) + + class DataModel: + name = str + + @elem_blueprint.get('/' + _name_api, strict_slashes=True) + @doc.summary("Show resources") + @doc.description("Display a listing of the resource.") + @doc.produces(content_type='application/json') + async def list(request): + return response.json(data_global_elements.get_interface(_name_api).gets()) + + @elem_blueprint.post('/' + _name_api, strict_slashes=True) + @doc.summary("Create new resource") + @doc.description("Store a newly created resource in storage.") + @doc.consumes(DataModel, location='body')#, required=True) + @doc.response_success(status=201, description='If successful created') + async def create(request): + return response.json(data_global_elements.get_interface(_name_api).post(request.json)) + + @elem_blueprint.post('/' + _name_api + "/find", strict_slashes=True) + @doc.summary("Create new resource if the name does not already exist") + @doc.description("Store a newly created resource in storage.") + @doc.consumes(DataModel, location='body')#, required=True) + @doc.response_success(status=201, description='If successful created') + async def find_with_name(request): + api = data_global_elements.get_interface(_name_api) + for elem in api.bdd: + if elem["name"] == request.json["name"]: + return response.json({"id": elem["id"]}) + raise ServerError("No data found", status_code=404) + + @elem_blueprint.get('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Show resources") + @doc.description("Display a listing of the resource.") + @doc.produces(content_type='application/json') + async def retrive(request, id): + value = data_global_elements.get_interface(_name_api).get(id) + if value != None: + return response.json(value) + raise ServerError("No data found", status_code=404) + + @elem_blueprint.get('/' + _name_api + '//video', strict_slashes=True) + @doc.summary("get videos list") + @doc.description("List all the videos availlable for this group.") + @doc.produces(content_type='application/json') + async def retrive_video(request, id): + value = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "group_id", id]], filter=["id"]) + if value != None: + return response.json(value) + raise ServerError("No data found", status_code=404) + + @elem_blueprint.get('/' + _name_api + '//video_no_saison', strict_slashes=True) + @doc.summary("get videos list who have no saison") + @doc.description("List all the videos availlable for this group tht does not depend on saison.") + @doc.produces(content_type='application/json') + async def retrive_video_no_saison(request, id): + value = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "group_id", id], ["==", "saison_id", None]], filter=["id"]) + if value != None: + return response.json(value) + raise ServerError("No data found", status_code=404) + + @elem_blueprint.get('/' + _name_api + '//saison', strict_slashes=True) + @doc.summary("get videos list who have no saison") + @doc.description("List all the videos availlable for this group tht does not depend on saison.") + @doc.produces(content_type='application/json') + async def retrive_saison(request, id): + value = data_global_elements.get_interface(data_global_elements.API_SAISON).gets_where(select=[["==", "group_id", id]], filter=["id"]) + if value != None: + return response.json(value) + raise ServerError("No data found", status_code=404) + + @elem_blueprint.put('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Update resource") + @doc.description("Update the specified resource in storage.") + @doc.response_success(status=201, description='If successful updated') + async def update(request, id): + ret = data_global_elements.get_interface(_name_api).put(id) + return response.json({}) + + @elem_blueprint.delete('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Remove resource") + @doc.description("Remove the specified resource from storage.") + @doc.response_success(status=201, description='If successful deleted') + async def delete(request, id): + ret = data_global_elements.get_interface(_name_api).delete(id) + if ret == True: + return response.json({}) + raise ServerError("No data found", status_code=404) + + _app.blueprint(elem_blueprint) diff --git a/back/src/api/root.py b/back/src/api/root.py new file mode 100644 index 0000000..a0141e7 --- /dev/null +++ b/back/src/api/root.py @@ -0,0 +1,43 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2019, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## + +import time +import json +import os +import sys +import datetime +import time, threading +import realog.debug as debug + +from sanic import Sanic +from sanic import response +from sanic import views +from sanic import Blueprint +from sanic.exceptions import ServerError + +from sanic_simple_swagger import swagger_blueprint, openapi_blueprint +from sanic_simple_swagger import doc + +import tools +import data_interface +import data_global_elements + +def add(_app): + @_app.route("/") + @doc.description("get api system information") + async def test(request): + return response.json({ + "api-type": "video-broker", + "api-version": _app.config['API_VERSION'], + "title": _app.config['API_TITLE'], + "description": _app.config['API_DESCRIPTION'], + "contact": _app.config['API_CONTACT_EMAIL'], + "licence": _app.config['API_LICENSE_NAME'] + }) diff --git a/back/src/api/saison.py b/back/src/api/saison.py new file mode 100644 index 0000000..3037169 --- /dev/null +++ b/back/src/api/saison.py @@ -0,0 +1,112 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2019, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## + +import time +import json +import os +import sys +import datetime +import time, threading +import realog.debug as debug + +from sanic import Sanic +from sanic import response +from sanic import views +from sanic import Blueprint +from sanic.exceptions import ServerError + +from sanic_simple_swagger import swagger_blueprint, openapi_blueprint +from sanic_simple_swagger import doc + +import tools +import data_interface +import data_global_elements + +def add(_app, _name_api): + elem_blueprint = Blueprint(_name_api) + + class DataModelBdd: + id = int + number = int + group_id = int + + data_global_elements.get_interface(_name_api).set_data_model(DataModelBdd) + + class DataModel: + number = int + group_id = int + + @elem_blueprint.get('/' + _name_api, strict_slashes=True) + @doc.summary("Show saisons") + @doc.description("Display a listing of the resource.") + @doc.produces(content_type='application/json') + async def list(request): + return response.json(data_global_elements.get_interface(_name_api).gets()) + + @elem_blueprint.post('/' + _name_api, strict_slashes=True) + @doc.summary("Create new saison") + @doc.description("Create a new saison for a aspecific group id.") + @doc.consumes(DataModel, location='body')#, required=True) + @doc.response_success(status=201, description='If successful created') + async def create(request): + return response.json(data_global_elements.get_interface(_name_api).post(request.json)) + + @elem_blueprint.post('/' + _name_api + "/find", strict_slashes=True) + @doc.summary("find a season existance") + @doc.description("return the ID of the season table.") + @doc.consumes(DataModel, location='body') + @doc.response_success(status=201, description='If successful created') + async def find_with_name(request): + api = data_global_elements.get_interface(_name_api) + for elem in api.bdd: + if elem["group_id"] == request.json["group_id"] \ + and elem["number"] == request.json["number"]: + return response.json({"id": elem["id"]}) + raise ServerError("No data found", status_code=404) + + @elem_blueprint.get('/' + _name_api + '//video', strict_slashes=True) + @doc.summary("Show videos") + @doc.description("List all the videos availlable for this group.") + @doc.produces(content_type='application/json') + async def retrive_video(request, id): + value = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "saison_id", id]], filter=["id"]) + if value != None: + return response.json(value) + raise ServerError("No data found", status_code=404) + + @elem_blueprint.get('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Show resources") + @doc.description("Display a listing of the resource.") + @doc.produces(content_type='application/json') + async def retrive(request, id): + value = data_global_elements.get_interface(_name_api).get(id) + if value != None: + return response.json(value) + raise ServerError("No data found", status_code=404) + + @elem_blueprint.put('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Update resource") + @doc.description("Update the specified resource in storage.") + @doc.response_success(status=201, description='If successful updated') + async def update(request, id): + ret = data_global_elements.get_interface(_name_api).put(id) + return response.json({}) + + @elem_blueprint.delete('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Remove resource") + @doc.description("Remove the specified resource from storage.") + @doc.response_success(status=201, description='If successful deleted') + async def delete(request, id): + ret = data_global_elements.get_interface(_name_api).delete(id) + if ret == True: + return response.json({}) + raise ServerError("No data found", status_code=404) + + _app.blueprint(elem_blueprint) diff --git a/back/src/api/type.py b/back/src/api/type.py new file mode 100644 index 0000000..001184c --- /dev/null +++ b/back/src/api/type.py @@ -0,0 +1,122 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2019, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## + +import time +import json +import os +import sys +import datetime +import time, threading +import realog.debug as debug + +from sanic import Sanic +from sanic import response +from sanic import views +from sanic import Blueprint +from sanic.exceptions import ServerError + +from sanic_simple_swagger import swagger_blueprint, openapi_blueprint +from sanic_simple_swagger import doc + +import tools +import data_interface +import data_global_elements + +def add(_app, _name_api): + elem_blueprint = Blueprint(_name_api) + + class DataModelBdd: + id = int + name = str + description = str + + data_global_elements.get_interface(_name_api).set_data_model(DataModelBdd) + + class DataModel: + name = str + description = str + + @elem_blueprint.get('/' + _name_api, strict_slashes=True) + @doc.summary("Show resources") + @doc.description("Display a listing of the resource.") + @doc.produces(content_type='application/json') + async def list(request): + return response.json(data_global_elements.get_interface(_name_api).gets()) + + @elem_blueprint.post('/' + _name_api, strict_slashes=True) + @doc.summary("Create new resource") + @doc.description("Store a newly created resource in storage.") + @doc.consumes(DataModel, location='body')#, required=True) + @doc.response_success(status=201, description='If successful created') + async def create(request): + return response.json(data_global_elements.get_interface(_name_api).post(request.json)) + + @elem_blueprint.get('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Show resources") + @doc.description("Display a listing of the resource.") + @doc.produces(content_type='application/json') + async def retrive(request, id): + value = data_global_elements.get_interface(_name_api).get(id) + if value != None: + return response.json(value) + raise ServerError("No data found", status_code=404) + + @elem_blueprint.put('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Update resource") + @doc.description("Update the specified resource in storage.") + @doc.response_success(status=201, description='If successful updated') + async def update(request, id): + ret = data_global_elements.get_interface(_name_api).put(id) + return response.json({}) + + @elem_blueprint.delete('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Remove resource") + @doc.description("Remove the specified resource from storage.") + @doc.response_success(status=201, description='If successful deleted') + async def delete(request, id): + ret = data_global_elements.get_interface(_name_api).delete(id) + if ret == True: + return response.json({}) + raise ServerError("No data found", status_code=404) + + @elem_blueprint.get('/' + _name_api + '//count', strict_slashes=True) + @doc.summary("Count resources in this cathegory") + @doc.description("count resources in this cathegory, in the whole tree.") + @doc.produces(content_type='application/json') + async def count_values(request, id): + count_value = data_global_elements.get_interface(data_global_elements.API_VIDEO).count(select=[["==", "type_id", id]]) + return response.json({"count":count_value}) + + + @elem_blueprint.get('/' + _name_api + '//video', strict_slashes=True) + @doc.summary("List the whole video ids") + @doc.description("List all video availlable with this type (list of ids).") + @doc.produces(content_type='application/json') + async def retrive_video(request, id): + list_values = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "type_id", id]], filter=["id"]) + return response.json(list_values) + + @elem_blueprint.get('/' + _name_api + '//video_no_group', strict_slashes=True) + @doc.summary("List the whole video ids") + @doc.description("List all video availlable with this type (list of ids).") + @doc.produces(content_type='application/json') + async def retrive_video_no_group(request, id): + list_values = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "type_id", id], ["==", "group_id", None]], filter=["id"]) + return response.json(list_values) + + @elem_blueprint.get('/' + _name_api + '//group', strict_slashes=True) + @doc.summary("List the whole video ids") + @doc.description("List all video availlable with this type (list of ids).") + @doc.produces(content_type='application/json') + async def retrive_group(request, id): + list_values = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "type_id", id], ["!=", "group_id", None]], filter=["group_id"]) + return response.json(list_values) + + _app.blueprint(elem_blueprint) \ No newline at end of file diff --git a/back/src/api/video.py b/back/src/api/video.py new file mode 100644 index 0000000..94d7182 --- /dev/null +++ b/back/src/api/video.py @@ -0,0 +1,156 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2019, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## + +import time +import json +import os +import sys +import copy +import datetime +import time, threading +import realog.debug as debug + +from sanic import Sanic +from sanic import response +from sanic import views +from sanic import Blueprint +from sanic.exceptions import ServerError + +from sanic_simple_swagger import swagger_blueprint, openapi_blueprint +from sanic_simple_swagger import doc + +import tools +import data_interface +import data_global_elements + +def generate_name(_value): + group_name = "" + if "group_id" in _value.keys(): + group_property = data_global_elements.get_interface(data_global_elements.API_GROUP).get(_value["group_id"]) + if group_property != None: + group_name = group_property["name"] + saison_number = "" + if "saison_id" in _value.keys(): + saison_property = data_global_elements.get_interface(data_global_elements.API_SAISON).get(_value["saison_id"]) + if saison_property != None: + saison_number = str(saison_property["number"]) + if len(saison_number) == 1: + saison_number = "0" + saison_number + out = "" + if group_name != "": + out += group_name + "-" + if saison_number != "": + out += "s" + saison_number + "-" + if "episode" in _value.keys() and _value["episode"] != None: + if _value["episode"] < 10: + out += "e00" + str(_value["episode"]) + "-" + elif _value["episode"] < 100: + out += "e0" + str(_value["episode"]) + "-" + else: + out += "e" + str(_value["episode"]) + "-" + out += _value["name"] + if "time" in _value.keys() and _value["time"] != None: + out += "(" + _value["name"] + ")" + return out + + +def add(_app, _name_api): + elem_blueprint = Blueprint(_name_api) + + class DataModelBdd: + id = int + sha512 = str + type_id = int + saison_id = [int, type(None)] + episode = [int, type(None)] + group_id = [int, type(None)] + name = str + description = [str, type(None)] + # creating time + create_date = str + # number of second + time = [int, type(None)] + + data_global_elements.get_interface(_name_api).set_data_model(DataModelBdd) + + class DataModel: + type_id = int + saison_id = int + episode = int + group_id = int + name = str + description = str + # creating time + create_date = str + # number of second + time = int + + @elem_blueprint.get('/' + _name_api, strict_slashes=True) + @doc.summary("Show saisons") + @doc.description("Display a listing of the resource.") + @doc.produces(content_type='application/json') + async def list(request): + return response.json(data_global_elements.get_interface(_name_api).gets()) + + @elem_blueprint.post('/' + _name_api, strict_slashes=True) + @doc.summary("Create new saison") + @doc.description("Create a new saison for a aspecific group id.") + @doc.consumes(DataModel, location='body')#, required=True) + @doc.response_success(status=201, description='If successful created') + async def create(request): + for type_key in ["sha512","type_id","name"]: + if type_key not in request.json.keys(): + raise ServerError("Bad Request: Missing Key '" + type_key + "'", status_code=400) + for type_key in ["date"]: + if type_key in request.json.keys(): + raise ServerError("Forbidden: Must not be set Key '" + type_key + "'", status_code=403) + for type_key in ["saison_id","episode","time","group_id","description"]: + if type_key not in request.json.keys(): + request.json[type_key] = None + request.json["create_date"] = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z' + #Find if already exist + list_elem = data_global_elements.get_interface(_name_api).find(["group_id", "sha512"], request.json); + for elem in list_elem: + return response.json(elem) + + return response.json(data_global_elements.get_interface(_name_api).post(request.json)) + + @elem_blueprint.get('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Show resources") + @doc.description("Display a listing of the resource.") + @doc.produces(content_type='application/json') + async def retrive(request, id): + value = data_global_elements.get_interface(_name_api).get(id) + if value != None: + generated_name = generate_name(value) + tmp = copy.deepcopy(value) + tmp["generated_name"] = generated_name + return response.json(tmp) + raise ServerError("No data found", status_code=404) + + @elem_blueprint.put('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Update resource") + @doc.description("Update the specified resource in storage.") + @doc.response_success(status=201, description='If successful updated') + async def update(request, id): + ret = data_global_elements.get_interface(_name_api).put(id) + return response.json({}) + + @elem_blueprint.delete('/' + _name_api + '/', strict_slashes=True) + @doc.summary("Remove resource") + @doc.description("Remove the specified resource from storage.") + @doc.response_success(status=201, description='If successful deleted') + async def delete(request, id): + ret = data_global_elements.get_interface(_name_api).delete(id) + if ret == True: + return response.json({}) + raise ServerError("No data found", status_code=404) + + _app.blueprint(elem_blueprint) diff --git a/back/src/app_video.py b/back/src/app_video.py new file mode 100755 index 0000000..6936a7d --- /dev/null +++ b/back/src/app_video.py @@ -0,0 +1,106 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2019, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## +#pip install flask --user +#pip install flask_restful --user +#pip install python-dateutil --user +#pip install sanic --user +#pip install sanic_simple_swagger --user + +from sanic import Sanic +from sanic import response +from sanic import views +from sanic import Blueprint +from sanic.exceptions import ServerError +from sanic_simple_swagger import swagger_blueprint, openapi_blueprint +from sanic_simple_swagger import doc +from spf import SanicPluginsFramework + +import dateutil.parser + + +import time +import json +import os +import sys +import datetime +import time, threading +import realog.debug as debug + +debug.enable_color() + +import tools +import data_interface +import data_global_elements + + +from sanic_cors.extension import cors +app = Sanic(__name__) +spf = SanicPluginsFramework(app) +spf.register_plugin(cors, automatic_options=True) + +app.config['API_VERSION'] = '1.0.0' +app.config['API_TITLE'] = 'Rest personal video API' +app.config['API_DESCRIPTION'] = 'Simple API for the Video broker.' +app.config['API_CONTACT_EMAIL'] = "yui.heero@gmail.com" +app.config['API_LICENSE_NAME'] = 'MPL 2.0' +app.config['API_LICENSE_URL'] = 'https://www.mozilla.org/en-US/MPL/2.0/' +app.config['schemes'] = ['http', 'https'] +if "REST_TMP_DATA" not in app.config.keys(): + app.config['REST_TMP_DATA'] = "tmp" +if "REST_MEDIA_DATA" not in app.config.keys(): + app.config['REST_MEDIA_DATA'] = os.path.join("data", "media") +if "REST_DATA" not in app.config.keys(): + app.config['REST_DATA'] = "data" +if "REST_HOST" not in app.config.keys(): + app.config['REST_HOST'] = "localhost" +if "REST_PORT" not in app.config.keys(): + app.config['REST_PORT'] = "80" + +app.blueprint(openapi_blueprint) +app.blueprint(swagger_blueprint) + + +def add_interface(_name): + data_global_elements.add_interface(_name, data_interface.DataInterface(_name, os.path.join(tools.get_run_path(), app.config['REST_DATA'], "bdd_" + _name + ".json"))) + +add_interface(data_global_elements.API_TYPE) +add_interface(data_global_elements.API_GROUP) +add_interface(data_global_elements.API_SAISON) +add_interface(data_global_elements.API_VIDEO) + +import api.root as api_root +api_root.add(app) + +import api.type as api_type +api_type.add(app, data_global_elements.API_TYPE) + +import api.group as api_group +api_group.add(app, data_global_elements.API_GROUP) + +import api.saison as api_saison +api_saison.add(app, data_global_elements.API_SAISON) + +import api.video as api_video +api_video.add(app, data_global_elements.API_VIDEO) + +import api.data as api_data +api_data.add(app, data_global_elements.API_DATA) + + + + +if __name__ == "__main__": + debug.info("Start REST application: " + str(app.config['REST_HOST']) + ":" + str(app.config['REST_PORT'])) + app.config.REQUEST_MAX_SIZE=10*1024*1024*1024 + app.run(host=app.config['REST_HOST'], port=int(app.config['REST_PORT'])) + debug.info("END program"); + sys.exit(0) + + diff --git a/back/src/data_global_elements.py b/back/src/data_global_elements.py new file mode 100644 index 0000000..be84e46 --- /dev/null +++ b/back/src/data_global_elements.py @@ -0,0 +1,41 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2019, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## + +interfaces = {} + +def get_list_interface(): + global interfaces + return interfaces + +def get_interface(_name): + global interfaces + return interfaces[_name] + +def add_interface(_name, _interface): + global interfaces + interfaces[_name] = _interface + + +import time, threading +def check_save(): + print(time.ctime()) + for elem in interfaces.keys(): + interfaces[elem].check_save() + threading.Timer(10, check_save).start() + +check_save() + + +API_TYPE = "type" +API_GROUP = "group" +API_SAISON = "saison" +API_VIDEO = "video" +API_DATA = "data" + diff --git a/back/src/data_interface.py b/back/src/data_interface.py new file mode 100644 index 0000000..64f4820 --- /dev/null +++ b/back/src/data_interface.py @@ -0,0 +1,246 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2019, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## + +import tools +import json +from realog import debug +import random +from sanic.exceptions import ServerError +## +## @breif Generic interface to access to the BDD (no BDD, direct file IO) +## +class DataInterface(): + def __init__(self, _name, _file): + self.model = None + self.name = _name + self.file = _file + self.bdd = [] + self.need_save = False + self.last_id = 0 + if tools.exist(self.file) == False: + self.mark_to_store() + self.last_id = random.randint(20, 100) + else: + data = tools.file_read_data(self.file) + self.bdd = json.loads(data) + self.upgrade_global_bdd_id(); + + def set_data_model(self, _data_model): + self.model = _data_model + + def check_with_model(self, _data): + if self.model == None: + return True + values = [] + for elem in dir(self.model): + if elem[:2] == "__": + continue + values.append(elem) + have_error = False + for key in _data.keys(): + if key not in values: + have_error = True + # TODO: ... + debug.warning("Add element that is not allowed " + key + " not in " + str(values)) + for elem in values: + if key not in _data.keys(): + have_error = True + # TODO: ... + debug.warning("Missing key " + elem + " not in " + str(_data.keys())) + if have_error == True: + return False + for key in _data.keys(): + elem = getattr(self.model, key) + if type(elem) == list: + find_error = True + for my_type in elem: + if type(_data[key]) == my_type: + find_error = False + break + if find_error == True: + debug.warning("data : " + str(_data)) + tmp_list = [] + for my_type in elem: + tmp_list.append(my_type.__name__) + debug.warning("[key='" + key + "'] try to add wrong type in BDD " + type(_data[key]).__name__ + " is not: " + str(my_type)) + else: + if type(_data[key]) != getattr(self.model, key): + debug.warning("data : " + str(_data)) + debug.warning("[key='" + key + "'] try to add wrong type in BDD " + type(_data[key]).__name__ + " is not: " + getattr(self.model, key).__name__) + return False + return True + + def upgrade_global_bdd_id(self): + for elem in self.bdd: + if 'id' not in elem.keys(): + continue + if elem["id"] >= self.last_id: + self.last_id = elem["id"] + 1 + + def get_table_index(self, _id): + id_in_bdd = 0 + for elem in self.bdd: + if 'id' in elem.keys() \ + and elem["id"] == _id: + return id_in_bdd + id_in_bdd += 1 + return None + + ## + ## @brief Mark the current BDD to store all in File system (sync) + ## + def mark_to_store(self): + self.need_save = True + + ## + ## @brief Check if the Bdd need to be stored. It is stored if it has been requested. + ## The BDD is store in a separate file and move in the old one. Safe way to store + ## + def check_save(self): + if self.need_save == False: + return + debug.warning("Save bdd: " + self.file) + data = json.dumps(self.bdd, sort_keys=True, indent=4) + self.need_save = False + tools.file_write_data_safe(self.file, data) + + def gets(self, filter=None): + debug.info("gets " + self.name) + if filter == None: + return self.bdd + return self.filter_object_values(self.bdd, filter) + + def gets_where(self, select, filter=None): + debug.info("gets " + self.name) + tmp_list = self.get_sub_list(self.bdd, select) + return self.filter_object_values(tmp_list, filter); + + def get(self, _id): + debug.info("get " + self.name + ": " + str(_id)) + for elem in self.bdd: + if 'id' in elem.keys() \ + and elem["id"] == _id: + return elem + return None + + def delete(self, _id): + debug.info("delete " + self.name + ": " + str(_id)) + id_in_bdd = self.get_table_index(_id) + if id_in_bdd == None: + return False + del self.bdd[id_in_bdd] + self.mark_to_store() + return True + + def put(self, _id, _value): + debug.info("put " + self.name + ": " + str(_id)) + id_in_bdd = self.get_table_index(_id) + if id_in_bdd == None: + return False + _value["id"] = _id + self.bdd[id_in_bdd] = _value + self.mark_to_store() + return True + + def post(self, _value): + debug.info("post " + self.name) + _value["id"] = self.last_id + self.last_id += 1 + if self.check_with_model(_value) == False: + raise ServerError("Corelation with BDD error", status_code=404) + self.bdd.append(_value) + self.mark_to_store() + return _value + + # TODO : rework this + def find(self, _list_token, _values): + out = [] + for elem in self.bdd: + find = True + for token in _list_token: + if elem[token] != _values[token]: + find = False + break + if find == True: + out.append(elem) + return out + + def count(self, select = None): + if select == None: + return len(self.bdd) + tmp = self.get_sub_list(self.bdd, select) + return len(tmp) + + def get_sub_list(self, _values, _select): + out = [] + for elem in _values: + find = True + if len(_select) == 0: + find = False + for elem_select in _select: + if len(elem_select) != 3: + raise ServerError("Internal Server Error: wrong select definition", 500) + type_check = elem_select[0] + token = elem_select[1] + value = elem_select[2] + if token in elem.keys(): + if type_check == "==": + if not (elem[token] == value): + find = False + break + elif type_check == "!=": + if not (elem[token] != value): + find = False + break + elif type_check == "<": + if not (elem[token] < value): + find = False + break + elif type_check == "<=": + if not (elem[token] <= value): + find = False + break + elif type_check == ">": + if not (elem[token] >= value): + find = False + break + elif type_check == ">=": + if not (elem[token] >= value): + find = False + break + else: + raise ServerError("Internal Server Error: unknow comparing type ...", 500) + else: + find = False + break + if find == True: + out.append(elem) + return out + + def filter_object_values(self, _values, _filter): + out = [] + if len(_filter) == 1: + token = _filter[0] + for elem in _values: + if token not in elem.keys(): + continue + if elem[token] not in out: + out.append(elem[token]) + return out + for elem in _values: + element_out = {} + for token in _filter: + if token not in elem.keys(): + continue + element_out[token] = elem[token] + out.append(element_out) + return out + + diff --git a/back/src/tools.py b/back/src/tools.py new file mode 100644 index 0000000..9851f86 --- /dev/null +++ b/back/src/tools.py @@ -0,0 +1,133 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2012, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## + +import os +import shutil +import errno +import fnmatch +import stat +# Local import +from realog import debug + +""" + +""" +def get_run_path(): + return os.getcwd() + +""" + +""" +def get_current_path(file): + return os.path.dirname(os.path.realpath(file)) + +def create_directory_of_file(file): + debug.info("Create directory of path: '" + file + "'") + path = os.path.dirname(file) + debug.info("Create directory: '" + path + "'") + try: + os.stat(path) + except: + os.makedirs(path) + +def get_list_sub_path(path): + # TODO : os.listdir(path) + for dirname, dirnames, filenames in os.walk(path): + return dirnames + return [] + +def remove_path_and_sub_path(path): + if os.path.isdir(path): + debug.verbose("remove path : '" + path + "'") + shutil.rmtree(path) + +def remove_file(path): + if os.path.isfile(path): + os.remove(path) + elif os.path.islink(path): + os.remove(path) + +def exist(path): + if os.path.isdir(path): + return True + if os.path.isfile(path): + return True + if os.path.islink(path): + return True + return False + +def file_size(path): + if not os.path.isfile(path): + return 0 + statinfo = os.stat(path) + return statinfo.st_size + +def file_read_data(path, binary=False): + debug.verbose("path= " + path) + if not os.path.isfile(path): + return "" + if binary == True: + file = open(path, "rb") + else: + file = open(path, "r") + data_file = file.read() + file.close() + return data_file + +def version_to_string(version): + version_ID = "" + for id in version: + if len(version_ID) != 0: + if type(id) == str: + version_ID += "-" + else: + version_ID += "." + version_ID += str(id) + return version_ID + +## +## @brief Write data in a specific path. +## @param[in] path Path of the data might be written. +## @param[in] data Data To write in the file. +## @param[in] only_if_new (default: False) Write data only if data is different. +## @return True Something has been copied +## @return False Nothing has been copied +## +def file_write_data(path, data, only_if_new=False): + if only_if_new == True: + if os.path.exists(path) == True: + old_data = file_read_data(path) + if old_data == data: + return False + #real write of data: + create_directory_of_file(path) + file = open(path, "w") + file.write(data) + file.close() + return True + +def file_write_data_safe(path, data): + #real write of data: + create_directory_of_file(path) + file = open(path + ".tmp", "w") + file.write(data) + file.close() + shutil.move(path + ".tmp", path) + return True + +def list_to_str(list): + if type(list) == type(str()): + return list + " " + else: + result = "" + # mulyiple imput in the list ... + for elem in list: + result += list_to_str(elem) + return result diff --git a/back/tools/sendFile.py b/back/tools/sendFile.py new file mode 100755 index 0000000..17df447 --- /dev/null +++ b/back/tools/sendFile.py @@ -0,0 +1,42 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2019, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## +import os +import sys +import requests # pip install requests + +class upload_in_chunks(object): + def __init__(self, filename, chunksize=1 << 13): + self.filename = filename + self.chunksize = chunksize + self.totalsize = os.path.getsize(filename) + self.readsofar = 0 + + def __iter__(self): + with open(self.filename, 'rb') as file: + while True: + data = file.read(self.chunksize) + if not data: + sys.stderr.write("\n") + break + self.readsofar += len(data) + percent = self.readsofar * 1e2 / self.totalsize + sys.stderr.write("\rSendfing data: {percent:3.0f}% {size:14.0f} / {total_size}".format(percent=percent, size=self.readsofar, total_size=self.totalsize)) + yield data + + def __len__(self): + return self.totalsize + +filename = 'Totally_Spies.mp4' + +result = requests.post("http://127.0.0.1:15080/data", data=upload_in_chunks(filename, chunksize=4096)) + + +print("result : " + str(result) + " " + result.text)#str(dir(result))) + diff --git a/back/tools/sendLocalData.py b/back/tools/sendLocalData.py new file mode 100755 index 0000000..10e1178 --- /dev/null +++ b/back/tools/sendLocalData.py @@ -0,0 +1,810 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +## +## @author Edouard DUPIN +## +## @copyright 2019, Edouard DUPIN, all right reserved +## +## @license MPL v2.0 (see license file) +## +import os +import copy +import sys +import hashlib +import requests # pip install requests +import realog.debug as debug +import magic +import json + +debug.enable_color(); + +class upload_in_chunks(object): + def __init__(self, filename, chunksize=1 + 13): + self.filename = filename + self.chunksize = chunksize + self.totalsize = os.path.getsize(filename) + self.readsofar = 0 + + def __iter__(self): + with open(self.filename, 'rb') as file: + while True: + data = file.read(self.chunksize) + if not data: + sys.stderr.write("\n") + break + self.readsofar += len(data) + percent = self.readsofar * 1e2 / self.totalsize + sys.stderr.write("\rSendfing data: {percent:3.0f}% {size:14.0f} / {total_size}".format(percent=percent, size=self.readsofar, total_size=self.totalsize)) + yield data + + def __len__(self): + return self.totalsize + +#filename = 'Totally_Spies.mp4' +#result = requests.post("http://127.0.0.1:15080/data", data=upload_in_chunks(filename, chunksize=4096)) +#debug.info("result : " + str(result) + " " + result.text)#str(dir(result))) + + +def extract_and_remove(_input_value, _start_mark, _stop_mark): + values = [] + out = "" + inside = False + inside_data = "" + for it in _input_value: + if inside == False \ + and it == _start_mark: + inside = True + elif inside == True \ + and it == _stop_mark: + inside = False + values.append(inside_data) + inside_data = "" + elif inside == True: + inside_data += it + else: + out += it + return (out, values) + +def create_directory_of_file(_file): + path = os.path.dirname(_file) + try: + os.stat(path) + except: + os.makedirs(path) + +## +## @brief Write data in a specific path. +## @param[in] path Path of the data might be written. +## @param[in] data Data To write in the file. +## @param[in] only_if_new (default: False) Write data only if data is different. +## @return True Something has been copied +## @return False Nothing has been copied +## +def file_write_data(_path, _data, _only_if_new=False): + if _only_if_new == True: + if os.path.exists(_path) == True: + old_data = file_read_data(_path) + if old_data == _data: + return False + #real write of data: + create_directory_of_file(_path) + file = open(_path, "w") + file.write(_data) + file.close() + return True + +def get_modify_time(_path): + return os.stat(_path).st_mtime + +def file_read_data(_path, _binary=False): + debug.verbose("path= " + _path) + if not os.path.isfile(_path): + return "" + if _binary == True: + file = open(_path, "rb") + else: + file = open(_path, "r") + data_file = file.read() + file.close() + return data_file + +def calculate_sha512(_path): + sha1 = hashlib.sha512() + file = open(_path, "rb") + while True: + body = file.read(4096) + sha1.update(body) + file.close() + return str(sha1.hexdigest()) + +def push_video_file(_path, _basic_key={}): + file_name, file_extension = os.path.splitext(_path); + debug.info("Send file: '" + file_name + "' with extention " + file_extension) + # internal file_extension ... + if file_extension == "sha512": + debug.verbose("file: '" + _path + "' sha512 extention ...") + return True + + debug.info("Add media : '" + _path + "'") + if file_extension[1:] not in ["avi", "mkv", "mov", "mp4", "ts"] \ + and file_name not in ["cover_1.jpg","cover_1.png", "cover_1.till", "cover_1.bmp", "cover_1.tga"]: + debug.warning("Not send file : " + _path + " Not manage file_extension... " + file_extension) + return False + + if file_name in ["cover_1.jpg","cover_1.png", "cover_1.till", "cover_1.bmp", "cover_1.tga"]: + # find a cover... + debug.warning("Not send cover Not managed ... : " + _path + " Not manage ...") + """ + debug.info("Send cover for: " + _basic_key["series-name"] + " " + _basic_key["saison"]); + if _basic_key["series-name"] == "": + debug.error(" ==> can not asociate at a specific seri"); + return False; + + etk::String groupName = _basic_key["series-name"]; + if _basic_key["saison"] != "": + groupName += ":" + _basic_key["saison"]; + + auto sending = _srv.setGroupCover(zeus::File::create(_path.getString(), ""), groupName); + sending.onSignal(progressCallback); + sending.waitFor(echrono::seconds(20000)); + """ + return True + + """ + if etk::path::exist(_path + ".sha512") == True: + debug.verbose("file sha512 exist ==> read it"); + uint64_t time_sha512 = get_modify_time(_path + ".sha512"); + uint64_t time_elem = get_modify_time(_path); + storedSha512_file = file_read_data(_path + ".sha512") + debug.verbose("file sha == " + storedSha512_file); + if time_elem > time_sha512: + debug.verbose("file time > sha time ==> regenerate new one ..."); + # check the current sha512 + storedSha512 = calculate_sha512(_path); + debug.verbose("calculated new sha'" + storedSha512 + "'"); + if storedSha512_file != storedSha512: + # need to remove the old sha file + auto idFileToRemove_fut = _srv.getId(storedSha512_file).waitFor(echrono::seconds(2)); + if idFileToRemove_fut.hasError() == True: + debug.error("can not remove the remote file with sha " + storedSha512_file); + else: + debug.info("Remove old deprecated file: " + storedSha512_file); + _srv.remove(idFileToRemove_fut.get()); + # note, no need to wait the call is async ... and the user does not interested with the result ... + + + # store new sha512 ==> this update tile too ... + file.open(etk::io::OpenMode::Write); + file.writeAll(storedSha512); + file.close(); + else: + # store new sha512 + /* + storedSha512 = file.readAllString(); + file.open(etk::io::OpenMode::Read); + file.writeAll(storedSha512); + file.close(); + */ + storedSha512 = storedSha512_file; + debug.verbose("read all sha from the file'" + storedSha512 + "'"); + + else: + """ + """ + if True: + storedSha512 = calculate_sha512(_path) + file_write_data(_path + ".sha512", storedSha512); + debug.info("calculate and store sha512 '" + storedSha512 + "'"); + debug.info("check file existance: sha='" + storedSha512 + "'"); + """ + + + # push only if the file exist + """ + # TODO : Check the metadata updating ... + auto idFile_fut = _srv.getId(storedSha512).waitFor(echrono::seconds(2)); + if idFile_fut.hasError() == False: + # media already exit ==> stop here ... + return True; + + # TODO: Do it better ==> add the calback to know the push progression ... + debug.verbose("Add File : " + _path + " sha='" + storedSha512 + "'"); + auto sending = _srv.add(zeus::File::create(_path, storedSha512)); + sending.onSignal(progressCallback); + debug.verbose("Add done ... now waiting ... "); + uint32_t mediaId = sending.waitFor(echrono::seconds(20000)).get(); + debug.verbose("END WAITING ... "); + if mediaId == 0: + debug.error("Get media ID = 0 With no error"); + return False; + + """ + mime = magic.Magic(mime=True) + mime_type = mime.from_file(_path) + headers_values = {'filename': _path, 'mime-type': mime_type} + result_send_data = requests.post("http://127.0.0.1:15080/data", headers=headers_values, data=upload_in_chunks(_path, chunksize=4096)) + debug.info("result *********** : " + str(result_send_data) + " " + result_send_data.text) + file_name = os.path.basename(file_name) + debug.info("Find file_name : '" + file_name + "'"); + # Remove Date (XXXX) or other titreadsofarle + file_name, dates = extract_and_remove(file_name, '(', ')'); + have_date = False + have_Title = False + for it in dates: + if len(it) == 0: + continue + if it[0] == '0' \ + or it[0] == '1' \ + or it[0] == '2' \ + or it[0] == '3' \ + or it[0] == '4' \ + or it[0] == '5' \ + or it[0] == '6' \ + or it[0] == '7' \ + or it[0] == '8' \ + or it[0] == '9': + # find a date ... + if have_date == True: + debug.info(" '" + file_name + "'") + debug.error("Parse Date error : () : " + it + " ==> multiple date") + continue + have_date = True + _basic_key["date"] = it + else: + if have_Title == True: + debug.info(" '" + file_name + "'") + debug.error("Parse Title error : () : " + it + " ==> multiple title") + continue + have_Title = True + # Other title + _basic_key.set["title2"] = it; + + # Remove the actors [XXX YYY][EEE TTT]... + file_name, actors = extract_and_remove(file_name, '[', ']'); + if len(actors) > 0: + debug.info(" '" + file_name + "'") + actor_list = [] + for it_actor in actors: + if actor_list != "": + actor_list += ";" + actor_list.append(it_actor) + _basic_key["actors"] = actor_list + list_element_base = file_name.split('-') + debug.warning("==> Title file: " + file_name) + debug.warning("==> Title cut : " + str(list_element_base)) + + list_element = []; + tmp_start_string = ""; + iii = 0 + while iii start elem: " + str(tmp_start_string)) + + if tmp_start_string != "": + list_element.append(tmp_start_string) + + debug.warning("==> list_element : " + str(list_element)) + + if len(list_element) == 1: + # nothing to do , it might be a film ... + _basic_key["title"] = list_element[0] + else: + if len(list_element) > 3 \ + and list_element[1][0] == 's' \ + and list_element[2][0] == 'e': + debug.warning("Parse format: xxx-sXX-eXX-kjhlkjlkj(1234).*") + # internal formalisme ... + saison = -1; + episode = -1; + series_name = list_element[0]; + + _basic_key["series-name"] = series_name + full_episode_name = list_element[3] + for yyy in range(4, len(list_element)): + full_episode_name += "-" + list_element[yyy] + + _basic_key["title"] = full_episode_name + if list_element[1][1:] == "XX": + # saison unknow ... ==> nothing to do ... + #saison = 123456789; + pass + else: + saison = int(list_element[1][1:]); + + if list_element[2][1:] == "XX": + # episode unknow ... ==> nothing to do ... + pass + else: + episode = int(list_element[2][1:]); + _basic_key["episode"] = int(episode) + + debug.info("Find a internal mode series: :"); + debug.info(" origin : '" + file_name + "'"); + saisonPrint = "XX"; + episodePrint = "XX"; + if saison < 0: + # nothing to do + pass + else: + saisonPrint = str(saison) + _basic_key["saison"] = saison + + if episode < 0: + # nothing to do + pass + elif episode < 10: + episodePrint = "0" + str(episode); + _basic_key["episode"] = episode + else: + episodePrint = str(episode); + _basic_key["episode"] = episode + + debug.info(" ==> '" + series_name + "-s" + saisonPrint + "-e" + episodePrint + "-" + full_episode_name + "'"); + elif len(list_element) > 2 \ + and list_element[1][0] == 'e': + debug.warning("Parse format: xxx-eXX-kjhlkjlkj(1234).*") + # internal formalisme ... + saison = -1; + episode = -1; + series_name = list_element[0]; + + _basic_key["series-name"] = series_name + full_episode_name = list_element[2] + for yyy in range(3, len(list_element)): + full_episode_name += "-" + list_element[yyy] + + _basic_key["title"] = full_episode_name + if list_element[1][1:] == "XX": + # episode unknow ... ==> nothing to do ... + pass + else: + episode = int(list_element[1][1:]); + _basic_key["episode"] = int(episode) + + debug.info("Find a internal mode series: :"); + debug.info(" origin : '" + file_name + "'"); + saisonPrint = "XX"; + episodePrint = "XX"; + if episode < 0: + # nothing to do + pass + elif episode < 10: + episodePrint = "0" + str(episode); + _basic_key["episode"] = episode + else: + episodePrint = str(episode); + _basic_key["episode"] = episode + + debug.info(" ==> '" + series_name + "-s" + saisonPrint + "-e" + episodePrint + "-" + full_episode_name + "'"); + + + result_send_data_json = json.loads(result_send_data.text) + debug.info("pared meta data: " + json.dumps(_basic_key, sort_keys=True, indent=4)) + data_model = { + "type_id": _basic_key["type"], + "sha512": result_send_data_json["sha512"], + #"group_id": int, + "name": _basic_key["title"], + # number of second + "time": None, + } + for elem in ["date", "description", "episode"]: #["actors", "date", "description", "episode", "title2"]: + if elem in _basic_key.keys(): + data_model[elem] = _basic_key[elem] + if "series-name" in _basic_key.keys(): + result_group_data = requests.post("http://127.0.0.1:15080/group/find", data=json.dumps({"name":_basic_key["series-name"]}, sort_keys=True, indent=4)) + debug.info("Create group ??? *********** : " + str(result_group_data) + " " + result_group_data.text) + if result_group_data.status_code == 404: + result_group_data = requests.post("http://127.0.0.1:15080/group", data=json.dumps({"name":_basic_key["series-name"]}, sort_keys=True, indent=4)) + debug.info("yes we create new group *********** : " + str(result_group_data) + " " + result_group_data.text) + group_id = result_group_data.json()["id"] + data_model["group_id"] = group_id + if "saison" in _basic_key.keys(): + result_saison_data = requests.post("http://127.0.0.1:15080/saison/find", data=json.dumps({"number":_basic_key["saison"], "group_id":group_id}, sort_keys=True, indent=4)) + debug.info("Create saison ??? *********** : " + str(result_saison_data) + " " + result_saison_data.text) + if result_saison_data.status_code == 404: + result_saison_data = requests.post("http://127.0.0.1:15080/saison", data=json.dumps({"number":_basic_key["saison"], "group_id":group_id}, sort_keys=True, indent=4)) + debug.info("yes we create new saison *********** : " + str(result_saison_data) + " " + result_saison_data.text) + saison_id = result_saison_data.json()["id"] + data_model["saison_id"] = saison_id + + result_send_data = requests.post("http://127.0.0.1:15080/video", data=json.dumps(data_model, sort_keys=True, indent=4)) + debug.info("result *********** : " + str(result_send_data) + " " + result_send_data.text) + + return True + + +def install_video_path( _path, _basic_key = {}): + debug.info("Parse : '" + _path + "'"); + list_sub_path = [fff for fff in os.listdir(_path) if os.path.isdir(os.path.join(_path, fff))] + for it_path in list_sub_path: + basic_key_tmp = copy.deepcopy(_basic_key) + debug.info("Add Sub path: '" + it_path + "'"); + if len(basic_key_tmp) == 0: + debug.info("find A '" + it_path + "' " + str(len(basic_key_tmp))); + if it_path == "documentary": + basic_key_tmp["type"] = 0 + elif it_path == "film": + basic_key_tmp["type"] = 1 + elif it_path == "film-annimation": + basic_key_tmp["type"] = 2 + elif it_path == "film-short": + basic_key_tmp["type"] = 3 + elif it_path == "tv-show": + basic_key_tmp["type"] = 4 + elif it_path == "tv-show-annimation": + basic_key_tmp["type"] = 5 + elif it_path == "theater": + basic_key_tmp["type"] = 6 + elif it_path == "one-man": + basic_key_tmp["type"] = 7 + elif it_path == "concert": + basic_key_tmp["type"] = 8 + elif it_path == "opera": + basic_key_tmp["type"] = 9 + else: + debug.info("find B '" + it_path + "' " + str(len(basic_key_tmp))) + if it_path == "saison_01": + basic_key_tmp["saison"] = 1 + elif it_path == "saison_02": + basic_key_tmp["saison"] = 2 + elif it_path == "saison_03": + basic_key_tmp["saison"] = 3 + elif it_path == "saison_04": + basic_key_tmp["saison"] = 4 + elif it_path == "saison_05": + basic_key_tmp["saison"] = 5 + elif it_path == "saison_06": + basic_key_tmp["saison"] = 6 + elif it_path == "saison_07": + basic_key_tmp["saison"] = 7 + elif it_path == "saison_08": + basic_key_tmp["saison"] = 8 + elif it_path == "saison_09": + basic_key_tmp["saison"] = 9 + elif it_path == "saison_10": + basic_key_tmp["saison"] = 10 + elif it_path == "saison_11": + basic_key_tmp["saison"] = 11 + elif it_path == "saison_12": + basic_key_tmp["saison"] = 12 + elif it_path == "saison_13": + basic_key_tmp["saison"] = 13 + elif it_path == "saison_14": + basic_key_tmp["saison"] = 14 + elif it_path == "saison_15": + basic_key_tmp["saison"] = 15 + elif it_path == "saison_16": + basic_key_tmp["saison"] = 16 + elif it_path == "saison_17": + basic_key_tmp["saison"] = 17 + elif it_path == "saison_18": + basic_key_tmp["saison"] = 18 + elif it_path == "saison_19": + basic_key_tmp["saison"] = 19 + elif it_path == "saison_20": + basic_key_tmp["saison"] = 20 + elif it_path == "saison_21": + basic_key_tmp["saison"] = 21 + elif it_path == "saison_22": + basic_key_tmp["saison"] = 22 + elif it_path == "saison_23": + basic_key_tmp["saison"] = 23 + elif it_path == "saison_24": + basic_key_tmp["saison"] = 24 + elif it_path == "saison_25": + basic_key_tmp["saison"] = 25 + elif it_path == "saison_26": + basic_key_tmp["saison"] = 26 + elif it_path == "saison_27": + basic_key_tmp["saison"] = 27 + elif it_path == "saison_28": + basic_key_tmp["saison"] = 28 + elif it_path == "saison_29": + basic_key_tmp["saison"] = 29 + else: + basic_key_tmp["series-name"] = it_path + debug.info("add a path " + os.path.join(_path, it_path) + " with keys " + str(basic_key_tmp)) + install_video_path(os.path.join(_path, it_path), basic_key_tmp); + + # Add files : + list_sub_file = [fff for fff in os.listdir(_path) if os.path.isfile(os.path.join(_path, fff))] + for it_file in list_sub_file: + basic_key_tmp = copy.deepcopy(_basic_key) + push_video_file(os.path.join(_path, it_file), basic_key_tmp); + + + + + +property = { + "hostname": "127.0.0.1", + "port": 15080, + "login": None, + "password": None, + +} + +import death.Arguments as arguments +import death.ArgElement as arg_element + +my_args = arguments.Arguments() +my_args.add_section("option", "Can be set one time in all case") +my_args.add("h", "help", desc="Display this help") +my_args.add("", "version", desc="Display the application version") +my_args.add("v", "verbose", list=[ + ["0","None"], + ["1","error"], + ["2","warning"], + ["3","info"], + ["4","debug"], + ["5","verbose"], + ["6","extreme_verbose"], + ], desc="display debug level (verbose) default =2") +my_args.add("a", "action", list=[ + ["tree","List all the files in a tree view ..."], + ["list","List all the files"], + ["push","push a single file"], + ["push_path","push a full folder"], + ], desc="possible action") +my_args.add("c", "color", desc="Display message in color") +my_args.add("f", "folder", haveParam=False, desc="Display the folder instead of the git repository name") +local_argument = my_args.parse() + +## +## @brief Display the help of this package. +## +def usage(): + color = debug.get_color_set() + # generic argument displayed : + my_args.display() + exit(0) + +## +## @brief Display the version of this package. +## +def version(): + color = debug.get_color_set() + import pkg_resources + debug.info("version: 0.0.0") + foldername = os.path.dirname(__file__) + debug.info("source folder is: " + foldername) + exit(0) + +folder = "dataPush" +requestAction = "list" + +# preparse the argument to get the verbose element for debug mode +def parse_arg(argument): + debug.warning("parse arg : " + argument.get_option_name() + " " + argument.get_arg()) + if argument.get_option_name() == "help": + usage() + return True + elif argument.get_option_name() == "version": + version() + return True + elif argument.get_option_name() == "verbose": + debug.set_level(int(argument.get_arg())) + return True + elif argument.get_option_name() == "color": + if check_boolean(argument.get_arg()) == True: + debug.enable_color() + else: + debug.disable_color() + return True + elif argument.get_option_name() == "folder": + folder = argument.get_arg() + return True + elif argument.get_option_name() == "action": + global requestAction + requestAction = argument.get_arg() + return True + return False + + +# parse default unique argument: +for argument in local_argument: + parse_arg(argument) + +debug.info("=================================="); +debug.info("== ZEUS test client start =="); +debug.info("=================================="); + + +def show_video(elem_video_id, indent): + indent_data = "" + while indent > 0: + indent_data += "\t" + indent -= 1 + result_video = requests.get("http://127.0.0.1:15080/video/" + str(elem_video_id) + "") + if result_video.status_code == 200: + video = result_video.json() + debug.info(indent_data + "- " + str(video["generated_name"])) + else: + debug.warning(indent_data + "get video id: " + str(elem_video_id) + " !!!!!! " + str(result_video.status_code) + "") + +# **************************************************************************************** +# ** Clear All the data base ... +# **************************************************************************************** +if requestAction == "clear": + debug.info("============================================"); + debug.info("== Clear data base: "); + debug.info("============================================"); + # TODO : Do it : + debug.error("NEED to add check in cmd line to execute it ..."); + """ + uint32_t count = remoteServiceVideo.count().wait().get(); + debug.debug("have " + count + " medias"); + for (uint32_t iii=0; iii " + tmpMax); + etk::Vector list = remoteServiceVideo.getIds(iii,tmpMax).wait().get(); + zeus::FutureGroup groupWait; + for (auto& it : list: + debug.info("remove ELEMENT : " + it); + groupWait.add(remoteServiceVideo.remove(it)); + groupWait.waitFor(echrono::seconds(2000)); + """ + debug.info("============================================"); + debug.info("== DONE =="); + debug.info("============================================"); +elif requestAction == "list": + debug.info("============================================"); + debug.info("== list files: "); + debug.info("============================================"); + list_types = requests.get("http://127.0.0.1:15080/type") + if list_types.status_code != 200: + debug.warning(" !! ca, ot get type list ... " + str(list_types.status_code) + "") + for elem in list_types.json(): + debug.info(" get type id: " + str(elem["id"])) + debug.info(" name: " + str(elem["name"])) + # get the count of video in this type + result_count = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/count") + if result_count.status_code == 200: + debug.info(" count: " + str(result_count.json()["count"])) + else: + debug.warning(" count: !!!!!! " + str(result_count.status_code) + "") + # get all the video list + result_video = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/video") + if result_video.status_code == 200: + if len(result_video.json()) != 0: + debug.info(" List video: " + str(result_video.json())) + else: + debug.warning(" List video: !!!!!! " + str(result_video.status_code) + "") + # get list of groups for this type + result_groups = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/group") + if result_groups.status_code == 200: + if len(result_groups.json()) != 0: + debug.info(" List group: " + str(result_groups.json())) + else: + debug.warning(" List group: !!!!!! " + str(result_groups.status_code) + "") + # get list of video without groups + result_video_solo = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/video_no_group") + if result_video_solo.status_code == 200: + if len(result_video_solo.json()) != 0: + debug.info(" List video solo: " + str(result_video_solo.json())) + else: + debug.warning(" List video solo: !!!!!! " + str(result_video_solo.status_code) + "") +elif requestAction == "tree": + debug.info("============================================"); + debug.info("== tree files: "); + debug.info("============================================"); + list_types = requests.get("http://127.0.0.1:15080/type") + if list_types.status_code != 200: + debug.warning(" !! ca, ot get type list ... " + str(list_types.status_code) + "") + for elem in list_types.json(): + debug.info("-------------------------------------------------") + debug.info(" " + str(elem["name"])) + debug.info("-------------------------------------------------") + # First get all the groups: + result_groups = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/group") + if result_groups.status_code == 200: + for elem_group_id in result_groups.json(): + result_group = requests.get("http://127.0.0.1:15080/group/" + str(elem_group_id) + "") + if result_group.status_code == 200: + group = result_group.json() + debug.info("\to- " + str(group["name"])) + # step 1: all the saison: + result_saison_in_group = requests.get("http://127.0.0.1:15080/group/" + str(elem_group_id) + "/saison") + if result_saison_in_group.status_code == 200: + for elem_saison_id in result_saison_in_group.json(): + result_saison = requests.get("http://127.0.0.1:15080/saison/" + str(elem_saison_id) + "") + if result_saison.status_code == 200: + debug.info("\t\t* saison " + str(result_saison.json()["number"])) + result_videos_in_saison = requests.get("http://127.0.0.1:15080/saison/" + str(result_saison.json()["id"]) + "/video") + if result_videos_in_saison.status_code == 200: + for elem_video_id in result_videos_in_saison.json(): + show_video(elem_video_id, 3) + else: + debug.warning("\t\tget video in saison id: " + str(elem_saison_id) + " !!!!!! " + str(result_videos_in_saison.status_code) + "") + show_video(elem_video_id, 2) + else: + debug.warning("\t\tget saison id: " + str(elem_saison_id) + " !!!!!! " + str(result_saison.status_code) + "") + else: + debug.warning("\t\tget saison in group id: " + str(elem_group_id) + " !!!!!! " + str(result_saison_in_group.status_code) + "") + # step 2: all the video with no saison: + result_videos_in_group = requests.get("http://127.0.0.1:15080/group/" + str(elem_group_id) + "/video_no_saison") + if result_videos_in_group.status_code == 200: + for elem_video_id in result_videos_in_group.json(): + show_video(elem_video_id, 2) + else: + debug.warning("\t\tget video in group id: " + str(elem_group_id) + " !!!!!! " + str(result_videos_in_group.status_code) + "") + else: + debug.warning("\tget group id: " + str(elem_group_id) + " !!!!!! " + str(result_group.status_code) + "") + else: + debug.warning("\t\tList group: !!!!!! " + str(result_groups.status_code) + "") + # get list of video without groups + result_video_solo = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/video_no_group") + if result_video_solo.status_code == 200: + for elem_video_id in result_video_solo.json(): + show_video(elem_video_id, 1) + else: + debug.warning("\t\tList video solo: !!!!!! " + str(result_video_solo.status_code) + "") + + + """ + uint32_t count = remoteServiceVideo.count().wait().get(); + debug.debug("have " + count + " medias"); + for (uint32_t iii=0; iii " + tmpMax); + etk::Vector list = remoteServiceVideo.getIds(iii, tmpMax).wait().get(); + for (auto& it : list: + # Get the media + zeus::ProxyMedia media = remoteServiceVideo.get(it).waitFor(echrono::seconds(2000)).get(); + if media.exist() == False: + debug.error("get media error"); + return -1; + debug.debug(" Get title ..."); + etk::String name = media.getMetadata("title").wait().get(); + debug.debug(" Get series-name ..."); + etk::String serie = media.getMetadata("series-name").wait().get(); + debug.debug(" Get episode ..."); + etk::String episode = media.getMetadata("episode").wait().get(); + debug.debug(" Get saison ..."); + etk::String saison = media.getMetadata("saison").wait().get(); + etk::String outputDesc = ""; + if serie != "": + outputDesc += serie + "-"; + if saison != "": + outputDesc += "s" + saison + "-"; + if episode != "": + outputDesc += "e" + episode + "-"; + outputDesc += name; + debug.info("[" + it + "] '" + outputDesc + "'"); + """ + debug.info("============================================"); + debug.info("== DONE =="); + debug.info("============================================"); +elif requestAction == "push": + debug.info("============================================"); + debug.info("== push file: "); + debug.info("============================================"); + push_video_file(folder); + debug.info("============================================"); + debug.info("== DONE =="); + debug.info("============================================"); +elif requestAction == "push_path": + debug.info("============================================"); + debug.info("== push path: "); + debug.info("============================================"); + install_video_path(folder); + debug.info("============================================"); + debug.info("== DONE =="); + debug.info("============================================"); +else: + debug.info("============================================"); + debug.error("== Unknow action: '" + requestAction + "'"); + debug.info("============================================");