HeeroYui/REST_video
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[DEV] update the whole application

Browse Source
This commit is contained in:
Edouard DUPIN 2019-12-19 00:18:52 +01:00
parent b9d44e648e
commit 7c41e3c391
8 changed files with 345 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/api/data.py
View File

@ -82,6 +82,8 @@ def add(_app, _name_api):
answer_data = {
"size": total_size,
"sha512": str(sha1.hexdigest()),
'filename': _request.headers["filename"],
'mime-type': _request.headers["mime-type"],
"already_exist": True,
}
@ -98,6 +100,8 @@ def add(_app, _name_api):
answer_data = {
"size": total_size,
"sha512": str(sha1.hexdigest()),
'filename': _request.headers["filename"],
'mime-type': _request.headers["mime-type"],
"already_exist": True,
}
await _response.write(json.dumps(answer_data, sort_keys=True, indent=4))

10
src/api/group.py
View File

@ -78,6 +78,16 @@ def add(_app, _name_api):
return response.json(value)
raise ServerError("No data found", status_code=404)
@elem_blueprint.get('/' + _name_api + '/<id:int>/video', strict_slashes=True)
@doc.summary("Show videos")
@doc.description("List all the videos availlable for this group.")
@doc.produces(content_type='application/json')
async def retrive(request, id):
value = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "group_id", id]], filter=["id"])
if value != None:
return response.json(value)
raise ServerError("No data found", status_code=404)
@elem_blueprint.put('/' + _name_api + '/<id:int>', strict_slashes=True)
@doc.summary("Update resource")
@doc.description("Update the specified resource in storage.")

33
src/api/type.py
View File

@ -86,4 +86,37 @@ def add(_app, _name_api):
return response.json({})
raise ServerError("No data found", status_code=404)
@elem_blueprint.get('/' + _name_api + '/<id:int>/count', strict_slashes=True)
@doc.summary("Count resources in this cathegory")
@doc.description("count resources in this cathegory, in the whole tree.")
@doc.produces(content_type='application/json')
async def count_values(request, id):
count_value = data_global_elements.get_interface(data_global_elements.API_VIDEO).count(select=[["==", "type_id", id]])
return response.json({"count":count_value})
@elem_blueprint.get('/' + _name_api + '/<id:int>/video', strict_slashes=True)
@doc.summary("List the whole video ids")
@doc.description("List all video availlable with this type (list of ids).")
@doc.produces(content_type='application/json')
async def count_values(request, id):
list_values = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "type_id", id]], filter=["id"])
return response.json(list_values)
@elem_blueprint.get('/' + _name_api + '/<id:int>/video_no_group', strict_slashes=True)
@doc.summary("List the whole video ids")
@doc.description("List all video availlable with this type (list of ids).")
@doc.produces(content_type='application/json')
async def count_values(request, id):
list_values = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "type_id", id], ["==", "group_id", None]], filter=["id"])
return response.json(list_values)
@elem_blueprint.get('/' + _name_api + '/<id:int>/group', strict_slashes=True)
@doc.summary("List the whole video ids")
@doc.description("List all video availlable with this type (list of ids).")
@doc.produces(content_type='application/json')
async def count_values(request, id):
list_values = data_global_elements.get_interface(data_global_elements.API_VIDEO).gets_where(select=[["==", "type_id", id], ["!=", "group_id", None]], filter=["group_id"])
return response.json(list_values)
_app.blueprint(elem_blueprint)

70
src/api/video.py
View File

@ -12,6 +12,7 @@ import time
import json
import os
import sys
import copy
import datetime
import time, threading
import realog.debug as debug
@ -29,6 +30,37 @@ import tools
import data_interface
import data_global_elements
def generate_name(_value):
group_name = ""
if "group_id" in _value.keys():
group_property = data_global_elements.get_interface(data_global_elements.API_GROUP).get(_value["group_id"])
if group_property != None:
group_name = group_property["name"]
saison_number = ""
if "saison_id" in _value.keys():
saison_property = data_global_elements.get_interface(data_global_elements.API_SAISON).get(_value["saison_id"])
if saison_property != None:
saison_number = str(saison_property["number"])
if len(saison_number) == 1:
saison_number = "0" + saison_number
out = ""
if group_name != "":
out += group_name + "-"
if saison_number != "":
out += "s" + saison_number + "-"
if "episode" in _value.keys() and _value["episode"] != None:
if _value["episode"] < 10:
out += "e00" + str(_value["episode"]) + "-"
elif _value["episode"] < 100:
out += "e0" + str(_value["episode"]) + "-"
else:
out += "e" + str(_value["episode"]) + "-"
out += _value["name"]
if "time" in _value.keys() and _value["time"] != None:
out += "(" + _value["name"] + ")"
return out
def add(_app, _name_api):
elem_blueprint = Blueprint(_name_api)
@ -36,13 +68,13 @@ def add(_app, _name_api):
id = int
sha512 = str
type_id = int
saison_id = int
saison_id = [int, type(None)]
episode = [int, type(None)]
group_id = int
group_id = [int, type(None)]
name = str
description = str
description = [str, type(None)]
# creating time
date = str
create_date = str
# number of second
time = [int, type(None)]
@ -56,7 +88,7 @@ def add(_app, _name_api):
name = str
description = str
# creating time
date = str
create_date = str
# number of second
time = int
@ -73,14 +105,21 @@ def add(_app, _name_api):
@doc.consumes(DataModel, location='body')#, required=True)
@doc.response_success(status=201, description='If successful created')
async def create(request):
"""
if "group_name" in request.json.keys():
id_group = data_global_elements.get_interface(API_GROUP).find_or_create_name(request.json["group_name"])
"""
if "episode" not in request.json.keys():
request.json["episode"] = None
else:
request.json["episode"] = int(request.json["episode"])
for type_key in ["sha512","type_id","name"]:
if type_key not in request.json.keys():
raise ServerError("Bad Request: Missing Key '" + type_key + "'", status_code=400)
for type_key in ["date"]:
if type_key in request.json.keys():
raise ServerError("Forbidden: Must not be set Key '" + type_key + "'", status_code=403)
for type_key in ["saison_id","episode","time","group_id","description"]:
if type_key not in request.json.keys():
request.json[type_key] = None
request.json["create_date"] = datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'
#Find if already exist
list_elem = data_global_elements.get_interface(_name_api).find(["group_id", "sha512"], request.json);
for elem in list_elem:
return response.json(elem)
return response.json(data_global_elements.get_interface(_name_api).post(request.json))
@elem_blueprint.get('/' + _name_api + '/<id:int>', strict_slashes=True)
@ -90,7 +129,10 @@ def add(_app, _name_api):
async def retrive(request, id):
value = data_global_elements.get_interface(_name_api).get(id)
if value != None:
return response.json(value)
generated_name = generate_name(value)
tmp = copy.deepcopy(value)
tmp["generated_name"] = generated_name
return response.json(tmp)
raise ServerError("No data found", status_code=404)
@elem_blueprint.put('/' + _name_api + '/<id:int>', strict_slashes=True)

23
src/app_video.py
View File

@ -65,33 +65,28 @@ app.blueprint(swagger_blueprint)
def add_interface(_name):
data_global_elements.add_interface(_name, data_interface.DataInterface(_name, os.path.join(tools.get_run_path(), app.config['REST_DATA'], "bdd_" + _name + ".json")))
API_TYPE = "type"
add_interface(API_TYPE)
API_GROUP = "group"
add_interface(API_GROUP)
API_SAISON = "saison"
add_interface(API_SAISON)
API_VIDEO = "video"
add_interface(API_VIDEO)
API_DATA = "data"
add_interface(data_global_elements.API_TYPE)
add_interface(data_global_elements.API_GROUP)
add_interface(data_global_elements.API_SAISON)
add_interface(data_global_elements.API_VIDEO)
import api.root as api_root
api_root.add(app)
import api.type as api_type
api_type.add(app, API_TYPE)
api_type.add(app, data_global_elements.API_TYPE)
import api.group as api_group
api_group.add(app, API_GROUP)
api_group.add(app, data_global_elements.API_GROUP)
import api.saison as api_saison
api_saison.add(app, API_SAISON)
api_saison.add(app, data_global_elements.API_SAISON)
import api.video as api_video
api_video.add(app, API_VIDEO)
api_video.add(app, data_global_elements.API_VIDEO)
import api.data as api_data
api_data.add(app, API_DATA)
api_data.add(app, data_global_elements.API_DATA)

7
src/data_global_elements.py
View File

@ -32,3 +32,10 @@ def check_save():
check_save()
API_TYPE = "type"
API_GROUP = "group"
API_SAISON = "saison"
API_VIDEO = "video"
API_DATA = "data"

153
src/data_interface.py
View File

@ -13,7 +13,9 @@ import json
from realog import debug
from sanic.exceptions import ServerError
##
## @breif Generic interface to access to the BDD (no BDD, direct file IO)
##
class DataInterface():
def __init__(self, _name, _file):
self.model = None
@ -23,7 +25,7 @@ class DataInterface():
self.need_save = False
self.last_id = 0
if tools.exist(self.file) == False:
self.need_save = True
self.mark_to_store()
else:
data = tools.file_read_data(self.file)
self.bdd = json.loads(data)
@ -81,15 +83,25 @@ class DataInterface():
if elem["id"] >= self.last_id:
self.last_id = elem["id"] + 1
def get_table_index(id):
def get_table_index(self, _id):
id_in_bdd = 0
for elem in self.bdd:
if 'id' in elem.keys() \
and elem["id"] == id:
and elem["id"] == _id:
return id_in_bdd
id_in_bdd += 1
return None
##
## @brief Mark the current BDD to store all in File system (sync)
##
def mark_to_store(self):
self.need_save = True
##
## @brief Check if the Bdd need to be stored. It is stored if it has been requested.
## The BDD is store in a separate file and move in the old one. Safe way to store
##
def check_save(self):
if self.need_save == False:
return
@ -98,45 +110,136 @@ class DataInterface():
self.need_save = False
tools.file_write_data_safe(self.file, data)
def gets(self):
def gets(self, filter=None):
debug.info("gets " + self.name)
return self.bdd
if filter == None:
return self.bdd
return self.filter_object_values(self.bdd, filter)
def get(self, id):
debug.info("get " + self.name + ": " + str(id))
def gets_where(self, select, filter=None):
debug.info("gets " + self.name)
tmp_list = self.get_sub_list(self.bdd, select)
return self.filter_object_values(tmp_list, filter);
def get(self, _id):
debug.info("get " + self.name + ": " + str(_id))
for elem in self.bdd:
if 'id' in elem.keys() \
and elem["id"] == id:
and elem["id"] == _id:
return elem
return None
def delete(self, id):
debug.info("delete " + self.name + ": " + str(id))
id_in_bdd = self.get_table_index(id)
def delete(self, _id):
debug.info("delete " + self.name + ": " + str(_id))
id_in_bdd = self.get_table_index(_id)
if id_in_bdd == None:
return False
del self.bdd[id_in_bdd]
self.need_save = True
self.mark_to_store()
return True
def put(self, id, value):
debug.info("put " + self.name + ": " + str(id))
id_in_bdd = self.get_table_index(id)
def put(self, _id, _value):
debug.info("put " + self.name + ": " + str(_id))
id_in_bdd = self.get_table_index(_id)
if id_in_bdd == None:
return False
value["id"] = id
self.bdd[id_in_bdd] = value
self.need_save = True
_value["id"] = _id
self.bdd[id_in_bdd] = _value
self.mark_to_store()
return True
def post(self, value):
def post(self, _value):
debug.info("post " + self.name)
value["id"] = self.last_id
_value["id"] = self.last_id
self.last_id += 1
if self.check_with_model(value) == False:
if self.check_with_model(_value) == False:
raise ServerError("Corelation with BDD error", status_code=404)
self.bdd.append(value)
self.need_save = True
return value
self.bdd.append(_value)
self.mark_to_store()
return _value
# TODO : rework this
def find(self, _list_token, _values):
out = []
for elem in self.bdd:
find = True
for token in _list_token:
if elem[token] != _values[token]:
find = False
break
if find == True:
out.append(elem)
return out
def count(self, select = None):
if select == None:
return len(self.bdd)
tmp = self.get_sub_list(self.bdd, select)
return len(tmp)
def get_sub_list(self, _values, _select):
out = []
for elem in _values:
find = True
if len(_select) == 0:
find = False
for elem_select in _select:
if len(elem_select) != 3:
raise ServerError("Internal Server Error: wrong select definition", 500)
type_check = elem_select[0]
token = elem_select[1]
value = elem_select[2]
if token in elem.keys():
if type_check == "==":
if not (elem[token] == value):
find = False
break
elif type_check == "!=":
if not (elem[token] != value):
find = False
break
elif type_check == "<":
if not (elem[token] < value):
find = False
break
elif type_check == "<=":
if not (elem[token] <= value):
find = False
break
elif type_check == ">":
if not (elem[token] >= value):
find = False
break
elif type_check == ">=":
if not (elem[token] >= value):
find = False
break
else:
raise ServerError("Internal Server Error: unknow comparing type ...", 500)
else:
find = False
break
if find == True:
out.append(elem)
return out
def filter_object_values(self, _values, _filter):
out = []
if len(_filter) == 1:
token = _filter[0]
for elem in _values:
if token not in elem.keys():
continue
if elem[token] not in out:
out.append(elem[token])
return out
for elem in _values:
element_out = {}
for token in _filter:
if token not in elem.keys():
continue
element_out[token] = elem[token]
out.append(element_out)
return out

111
tools/sendLocalData.py
View File

@ -16,6 +16,8 @@ import realog.debug as debug
import magic
import json
debug.enable_color();
class upload_in_chunks(object):
def __init__(self, filename, chunksize=1 + 13):
self.filename = filename
@ -40,7 +42,7 @@ class upload_in_chunks(object):
#filename = 'Totally_Spies.mp4'
#result = requests.post("http://127.0.0.1:15080/data", data=upload_in_chunks(filename, chunksize=4096))
#print("result : " + str(result) + " " + result.text)#str(dir(result)))
#debug.info("result : " + str(result) + " " + result.text)#str(dir(result)))
def extract_and_remove(_input_value, _start_mark, _stop_mark):
@ -221,7 +223,7 @@ def push_video_file(_path, _basic_key={}):
mime_type = mime.from_file(_path)
headers_values = {'filename': _path, 'mime-type': mime_type}
result_send_data = requests.post("http://127.0.0.1:15080/data", headers=headers_values, data=upload_in_chunks(_path, chunksize=4096))
print("result *********** : " + str(result_send_data) + " " + result_send_data.text)
debug.info("result *********** : " + str(result_send_data) + " " + result_send_data.text)
file_name = os.path.basename(file_name)
debug.info("Find file_name : '" + file_name + "'");
# Remove Date (XXXX) or other titreadsofarle
@ -324,7 +326,7 @@ def push_video_file(_path, _basic_key={}):
pass
else:
episode = int(list_element[2][1:]);
_basic_key["episode"] = episode
_basic_key["episode"] = int(episode)
debug.info("Find a internal mode series: :");
debug.info(" origin : '" + file_name + "'");
@ -345,10 +347,10 @@ def push_video_file(_path, _basic_key={}):
pass
elif episode < 10:
episodePrint = "0" + str(episode);
_basic_key["episode"] = str(episode)
_basic_key["episode"] = episode
else:
episodePrint = str(episode);
_basic_key["episode"] = str(episode)
_basic_key["episode"] = episode
debug.info(" ==> '" + series_name + "-s" + saisonPrint + "-e" + episodePrint + "-" + full_episode_name + "'");
@ -363,28 +365,28 @@ def push_video_file(_path, _basic_key={}):
# number of second
"time": None,
}
for elem in ["actors", "date", "description", "episode", "title2"]:
for elem in ["date", "description", "episode"]: #["actors", "date", "description", "episode", "title2"]:
if elem in _basic_key.keys():
data_model[elem] = _basic_key[elem]
if "series-name" in _basic_key.keys():
result_group_data = requests.post("http://127.0.0.1:15080/group/find", data=json.dumps({"name":_basic_key["series-name"]}, sort_keys=True, indent=4))
print("Create group ??? *********** : " + str(result_group_data) + " " + result_group_data.text)
debug.info("Create group ??? *********** : " + str(result_group_data) + " " + result_group_data.text)
if result_group_data.status_code == 404:
result_group_data = requests.post("http://127.0.0.1:15080/group", data=json.dumps({"name":_basic_key["series-name"]}, sort_keys=True, indent=4))
print("yes we create new group *********** : " + str(result_group_data) + " " + result_group_data.text)
debug.info("yes we create new group *********** : " + str(result_group_data) + " " + result_group_data.text)
group_id = result_group_data.json()["id"]
data_model["group_id"] = group_id
if "saison" in _basic_key.keys():
result_saison_data = requests.post("http://127.0.0.1:15080/saison/find", data=json.dumps({"number":int(_basic_key["saison"]), "group_id":group_id}, sort_keys=True, indent=4))
print("Create saison ??? *********** : " + str(result_saison_data) + " " + result_saison_data.text)
debug.info("Create saison ??? *********** : " + str(result_saison_data) + " " + result_saison_data.text)
if result_saison_data.status_code == 404:
result_saison_data = requests.post("http://127.0.0.1:15080/saison", data=json.dumps({"number":int(_basic_key["saison"]), "group_id":group_id}, sort_keys=True, indent=4))
print("yes we create new saison *********** : " + str(result_saison_data) + " " + result_saison_data.text)
debug.info("yes we create new saison *********** : " + str(result_saison_data) + " " + result_saison_data.text)
saison_id = result_saison_data.json()["id"]
data_model["saison_id"] = saison_id
result_send_data = requests.post("http://127.0.0.1:15080/video", data=json.dumps(data_model, sort_keys=True, indent=4))
print("result *********** : " + str(result_send_data) + " " + result_send_data.text)
debug.info("result *********** : " + str(result_send_data) + " " + result_send_data.text)
return True
@ -517,6 +519,7 @@ my_args.add("v", "verbose", list=[
["6","extreme_verbose"],
], desc="display debug level (verbose) default =2")
my_args.add("a", "action", list=[
["tree","List all the files in a tree view ..."],
["list","List all the files"],
["push","push a single file"],
["push_path","push a full folder"],
@ -540,9 +543,9 @@ def usage():
def version():
color = debug.get_color_set()
import pkg_resources
print("version: 0.0.0")
debug.info("version: 0.0.0")
foldername = os.path.dirname(__file__)
print("source folder is: " + foldername)
debug.info("source folder is: " + foldername)
exit(0)
folder = "dataPush"
@ -613,6 +616,88 @@ elif requestAction == "list":
debug.info("============================================");
debug.info("== list files: ");
debug.info("============================================");
list_types = requests.get("http://127.0.0.1:15080/type")
if list_types.status_code != 200:
debug.warning(" !! ca, ot get type list ... " + str(list_types.status_code) + "")
for elem in list_types.json():
debug.info(" get type id: " + str(elem["id"]))
debug.info(" name: " + str(elem["name"]))
# get the count of video in this type
result_count = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/count")
if result_count.status_code == 200:
debug.info(" count: " + str(result_count.json()["count"]))
else:
debug.warning(" count: !!!!!! " + str(result_count.status_code) + "")
# get all the video list
result_video = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/video")
if result_video.status_code == 200:
if len(result_video.json()) != 0:
debug.info(" List video: " + str(result_video.json()))
else:
debug.warning(" List video: !!!!!! " + str(result_video.status_code) + "")
# get list of groups for this type
result_groups = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/group")
if result_groups.status_code == 200:
if len(result_groups.json()) != 0:
debug.info(" List group: " + str(result_groups.json()))
else:
debug.warning(" List group: !!!!!! " + str(result_groups.status_code) + "")
# get list of video without groups
result_video_solo = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/video_no_group")
if result_video_solo.status_code == 200:
if len(result_video_solo.json()) != 0:
debug.info(" List video solo: " + str(result_video_solo.json()))
else:
debug.warning(" List video solo: !!!!!! " + str(result_video_solo.status_code) + "")
elif requestAction == "tree":
debug.info("============================================");
debug.info("== tree files: ");
debug.info("============================================");
list_types = requests.get("http://127.0.0.1:15080/type")
if list_types.status_code != 200:
debug.warning(" !! ca, ot get type list ... " + str(list_types.status_code) + "")
for elem in list_types.json():
debug.info("-------------------------------------------------")
debug.info(" " + str(elem["name"]))
debug.info("-------------------------------------------------")
# First get all the groups:
result_groups = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/group")
if result_groups.status_code == 200:
for elem_group_id in result_groups.json():
result_group = requests.get("http://127.0.0.1:15080/group/" + str(elem_group_id) + "")
if result_group.status_code == 200:
group = result_group.json()
debug.info(" o- " + str(group["name"]))
# step 1: all the video:
result_videos_in_group = requests.get("http://127.0.0.1:15080/group/" + str(elem_group_id) + "/video")
if result_videos_in_group.status_code == 200:
for elem_video_id in result_videos_in_group.json():
result_video = requests.get("http://127.0.0.1:15080/video/" + str(elem_video_id) + "")
if result_video.status_code == 200:
video = result_video.json()
debug.info(" - " + str(video["generated_name"]))
else:
debug.warning(" get video id: " + str(elem_video_id) + " !!!!!! " + str(result_video.status_code) + "")
else:
debug.warning(" get video in group id: " + str(elem_group_id) + " !!!!!! " + str(result_videos_in_group.status_code) + "")
else:
debug.warning(" get group id: " + str(elem_group_id) + " !!!!!! " + str(result_group.status_code) + "")
else:
debug.warning(" List group: !!!!!! " + str(result_groups.status_code) + "")
# get list of video without groups
result_video_solo = requests.get("http://127.0.0.1:15080/type/" + str(elem["id"]) + "/video_no_group")
if result_video_solo.status_code == 200:
for elem_video_id in result_video_solo.json():
result_video = requests.get("http://127.0.0.1:15080/video/" + str(elem_video_id) + "")
if result_video.status_code == 200:
video = result_video.json()
debug.info(" - " + str(video["generated_name"]))
else:
debug.warning(" get video id: " + str(elem_video_id) + " !!!!!! " + str(result_video.status_code) + "")
else:
debug.warning(" List video solo: !!!!!! " + str(result_video_solo.status_code) + "")
"""
uint32_t count = remoteServiceVideo.count().wait().get();
debug.debug("have " + count + " medias");