kangaroo-and-rabbit/karanage
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

[DEV] update to support mypy

Browse Source
This commit is contained in:
Edouard DUPIN 2023-02-21 23:13:20 +01:00
parent 10d6edad5a
commit 11935aa710
21 changed files with 202 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
client/python/karanage/karanage/connection.py
View File

@ -14,6 +14,7 @@ import json
from datetime import datetime, timedelta
from pathlib import Path
from typing import (
Any,
Dict,
List,
NamedTuple,
@ -21,6 +22,7 @@ from typing import (
)
import time
import requests
from .exception import KaranageException
class KaranageResponse:
@ -42,8 +44,8 @@ class KaranageConnectionInterface(ABC):
def post(
self,
service: str,
url_offset: str,
data: Optional[Dict] = None,
url_offset: Optional[str],
data: Any = None,
headers: Optional[Dict] = None,
params: Optional[Dict] = None,
) -> KaranageResponse:
@ -61,7 +63,7 @@ class KaranageConnectionInterface(ABC):
def get(
self,
service: str,
url_offset: str,
url_offset: Optional[str],
headers: Optional[Dict] = None,
params: Optional[Dict] = None,
) -> KaranageResponse:
@ -87,7 +89,7 @@ class KaranageConnection(KaranageConnectionInterface):
group: Optional[str] = None,
token: Optional[str] = None,
config_file: Optional[str] = None,
default_values: Optional[str] = None,
default_values: Optional[Dict[str, str]] = None,
) -> None:
"""Initialize the communication class.
:param url: URL of the karanage API server.
@ -161,8 +163,8 @@ class KaranageConnection(KaranageConnectionInterface):
def post(
self,
service: str,
url_offset: str,
data: Optional[Dict] = None,
url_offset: Optional[str],
data: Any = None,
headers: Optional[Dict] = None,
params: Optional[Dict] = None,
) -> KaranageResponse:
@ -178,7 +180,7 @@ class KaranageConnection(KaranageConnectionInterface):
def get(
self,
service: str,
url_offset: str,
url_offset: Optional[str],
headers: Optional[Dict] = None,
params: Optional[Dict] = None,
) -> KaranageResponse:
@ -196,13 +198,12 @@ class MockData(NamedTuple):
:param NamedTuple: Tuple generic interface
"""
type: str
service: str
url_offset: str
data: Dict
headers: Dict
params: Dict
url_offset: Optional[str]
data: Any
headers: Optional[Dict[str, Any]]
params: Optional[Dict[str, Any]]
class KaranageMock(KaranageConnectionInterface):
@ -221,10 +222,10 @@ class KaranageMock(KaranageConnectionInterface):
def post(
self,
service: str,
url_offset: str,
data: Optional[Dict] = None,
headers: Optional[Dict] = None,
params: Optional[Dict] = None,
url_offset: Optional[str],
data: Any = None,
headers: Optional[Dict[str, Any]] = None,
params: Optional[Dict[str, Any]] = None,
) -> KaranageResponse:
self.request.append(
MockData("POST", service, url_offset, data, headers, params)
@ -234,14 +235,14 @@ class KaranageMock(KaranageConnectionInterface):
def get(
self,
service: str,
url_offset: str,
headers: Optional[Dict] = None,
params: Optional[Dict] = None,
url_offset: Optional[str],
headers: Optional[Dict[str, Any]] = None,
params: Optional[Dict[str, Any]] = None,
) -> KaranageResponse:
self.request.append(MockData("GET", service, url_offset, None, headers, params))
return KaranageResponse(f"{service}/{url_offset}", 200, "{}")
def get_values(self, time_out: Optional[float] = None) -> List[KaranageResponse]:
def get_values(self, time_out: Optional[float] = None) -> List[MockData]:
"""get the list of last added values
:param time_out: Timeout before exiting in error

6
client/python/karanage/karanage/exception.py
View File

@ -7,10 +7,10 @@
##
## @license MPL v2.0 (see license file)
##
from typing import Optional
class KaranageException(Exception):
def __init__(self, message, error_id, error_message):
def __init__(self, message:str, error_id: Optional[int] = None, error_message: Optional[str] = None):
# Call the base class constructor with the parameters it needs
super().__init__(message)
@ -19,4 +19,4 @@ class KaranageException(Exception):
self.error_message = error_message
def __str__(self):
return f"{Exception.__str__(self)} Status={self.error_id} message='{self.error_message}'"
return f"{Exception.__str__(self)} status={self.error_id} message='{self.error_message}'"

39
client/python/karanage/karanage/log.py
View File

@ -11,6 +11,7 @@ from datetime import datetime, timezone
import enum
import json
from typing import (
Any,
Dict,
List,
Optional,
@ -23,7 +24,7 @@ from .exception import KaranageException
class GroupLogElement:
def __init__(self, id: int, data: str, time: datetime = None):
def __init__(self, id: int, data: str, time: Optional[datetime] = None):
self.id = id
self.time = time
self.data = data
@ -54,7 +55,7 @@ class KaranageLog:
:param uuid_group: local internal group UUID
:param time_log: Receive time of the log
"""
param = {}
param: Dict[str, Any] = {}
if id is not None:
param["id"] = id
if uuid_group is not None:
@ -62,7 +63,10 @@ class KaranageLog:
if time is not None:
param["time"] = time.astimezone(timezone.utc).isoformat()
ret = self.connection.post(self.service, self.system, data=data, params=param)
if not 200 <= ret.status <= 299:
if ret is None:
raise KaranageException(
f"Fail send message sub-library return None")
if ret.status is None or not 200 <= ret.status <= 299:
raise KaranageException(
f"Fail send message: '{ret.url}'", ret.status, ret.data
)
@ -76,13 +80,23 @@ class KaranageLog:
"""
data = []
for elem in data_input:
data.append(
{
"id": elem.id,
"time": elem.time.astimezone(timezone.utc).isoformat(),
"data": elem.data,
}
)
if elem.time is not None:
data.append(
{
"id": elem.id,
"time": elem.time.astimezone(timezone.utc).isoformat(),
"data": elem.data,
}
)
else:
data.append(
{
"id": elem.id,
"time": datetime.now().astimezone(timezone.utc).isoformat(),
"data": elem.data,
}
)
param = {}
if uuid_group is not None:
@ -90,7 +104,10 @@ class KaranageLog:
ret = self.connection.post(
self.service, f"{self.system}/push_multiple", data=data, params=param
)
if not 200 <= ret.status <= 299:
if ret is None:
raise KaranageException(
f"Fail send message sub-library return None")
if ret.status is None or not 200 <= ret.status <= 299:
raise KaranageException(
f"Fail send message: '{ret.url}'", ret.status, ret.data
)

0
client/python/karanage-tools/karanage-tools/__init__.py → client/python/karanage/karanage/py.typed
View File

44
client/python/karanage/karanage/state.py
View File

@ -9,7 +9,7 @@
##
import enum
import json
from typing import Dict, Optional, Union, List, NamedTuple
from typing import Any, Dict, Optional, Union, List, NamedTuple
from datetime import datetime, timezone
import requests
@ -64,7 +64,10 @@ class KaranageState:
if state is not None:
param["state"] = state.value
ret = self.connection.post("state", topic, data=data, params=param)
if not 200 <= ret.status <= 299:
if ret is None:
raise KaranageException(
f"Fail send message sub-library return None")
if ret.status is None or not 200 <= ret.status <= 299:
raise KaranageException(
f"Fail send message: '{ret.url}'", ret.status, ret.data
)
@ -78,15 +81,22 @@ class KaranageState:
"mode":"raw" # request raw mode to have the timestant in a float in second
}
if since is not None:
if since is str:
if type(since) is str:
param["since"] = since
else:
if type(since) is datetime:
param["since"] = since.replace(tzinfo=timezone.utc).isoformat()
else:
raise KaranageException(
f"Wrong input parameter type Must be a str or datetime: '{type(since)}'")
ret = self.connection.get("state", topic, params=param)
if 200 <= ret.status <= 299:
if ret is None:
raise KaranageException(
f"Fail send message sub-library return None")
if ret.status is not None and 200 <= ret.status <= 299:
out: List[StateData] = []
for elem in json.loads(ret.data):
out.append(convert_to_state_data(elem, topic))
if ret.data is not None:
for elem in json.loads(ret.data):
out.append(convert_to_state_data(elem, topic))
return out
raise KaranageException(f"Fail get data: '{ret.url}'", ret.status, ret.data)
@ -103,23 +113,29 @@ class KaranageState:
:param limit: the number of value we want to get
:return: A dictionary with the requested data.
"""
param = {
param:Dict[str, Any] = {
"mode":"raw" # request raw mode to have the timestant in a float in second
}
if since is not None:
if since is str:
if type(since) is str:
param["since"] = since
else:
if type(since) is datetime:
param["since"] = since.replace(tzinfo=timezone.utc).isoformat()
else:
raise KaranageException(
f"Wrong input parameter type Must be a str or datetime: '{type(since)}'")
if since_id is not None:
param["sinceId"] = since_id
if limit is not None:
param["limit"] = limit
ret = self.connection.get("state_history", topic, params=param)
if 200 <= ret.status <= 299:
if ret is None:
raise KaranageException(
f"Fail send message sub-library return None")
if ret.status is not None and 200 <= ret.status <= 299:
out: List[StateData] = []
for elem in json.loads(ret.data):
out.append(convert_to_state_data(elem, topic))
if ret.data is not None:
for elem in json.loads(ret.data):
out.append(convert_to_state_data(elem, topic))
return out
raise KaranageException(f"Fail get data: '{ret.url}'", ret.status, ret.data)

4
client/python/karanage/setup.py
View File

@ -53,10 +53,10 @@ setup(
zip_safe=False,
)
# To developp: sudo ./setup.py install --user
# To develop: sudo ./setup.py install --user
# sudo ./setup.py develop --user
# TO register all in pip: ./setup.py register sdist upload
# To register:
# https://packaging.python.org/en/latest/tutorials/packaging-projects/
# python3 -m build
# python3 -m twine upload dist/*

0
client/python/karanage-tools/.flake8 → client/python/karanage_tools/.flake8
View File

0
client/python/karanage-tools/.isort.cfg → client/python/karanage_tools/.isort.cfg
View File

0
client/python/karanage-tools/COPYING → client/python/karanage_tools/COPYING
View File

0
client/python/karanage-tools/MANIFEST.in → client/python/karanage_tools/MANIFEST.in
View File

0
client/python/karanage-tools/README.md → client/python/karanage_tools/README.md
View File

2
client/python/karanage_tools/bin/karanage-system Executable file
View File

@ -0,0 +1,2 @@
#!/bin/python3
from karanage_tools import karanage_system

2
client/python/karanage_tools/bin/karanage-tools-state-get Executable file
View File

@ -0,0 +1,2 @@
#!/bin/python3
from karanage_tools import karanage_tools_state_get

2
client/python/karanage_tools/bin/karanage-tools-state-history-get Executable file
View File

@ -0,0 +1,2 @@
#!/bin/python3
from karanage_tools import karanage_tools_state_history_get

9
client/python/karanage_tools/karanage_tools/__init__.py Normal file
View File

@ -0,0 +1,9 @@
#!/usr/bin/python3
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2023, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##

82
client/python/karanage-tools/bin/karanage-system → client/python/karanage_tools/karanage_tools/karanage_system.py
View File

@ -1,11 +1,18 @@
#!/bin/python3
# Importing the library
#!/usr/bin/python3
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2023, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import argparse
import json
from pathlib import Path
import subprocess
import time
from typing import Dict, List
from typing import Any, Dict, List, Optional
from karanage import (
KaranageConnection,
@ -19,6 +26,7 @@ import psutil
cpu_core_count = psutil.cpu_count(logical=False)
cpu_thread_count = psutil.cpu_count()
def get_mounts() -> Dict:
process = subprocess.Popen(
["mount", "-v"], stdout=subprocess.PIPE, stderr=subprocess.PIPE
@ -39,21 +47,24 @@ def get_mounts() -> Dict:
tmp = line.split(" ")
if len(tmp) <= 2:
continue
if tmp[0].startswith('/dev/'):
if tmp[0].startswith("/dev/"):
out[tmp[2]] = tmp[0][5:]
return out
mounting_points = get_mounts();
mounting_points = get_mounts()
def filter(data: Dict, filter_list: List[str]) -> Dict:
out = {}
#print(f"Request filter {data.keys()} with filter {filter_list}")
# print(f"Request filter {data.keys()} with filter {filter_list}")
for key in data:
if key in filter_list:
out[key] = data[key]
return out
def need_process(data: Dict) -> Dict:
def need_process(data: Dict[str, Any]) -> Any:
return configuration["cpu"] == "auto" or "include" in configuration["cpu"]
@ -69,6 +80,7 @@ def create_cpu_data() -> Dict:
"max": cpu_frequency.max,
}
def create_memory_data() -> Dict:
# svmem(total=14473519104, available=8289726464, percent=42.7, used=5380497408, free=3276263424, active=1775763456, inactive=8335540224, buffers=361771008, cached=5454987264, shared=243720192, slab=544526336)
mem = psutil.virtual_memory()
@ -77,6 +89,7 @@ def create_memory_data() -> Dict:
"total": mem.total,
}
def create_swap_data() -> Dict:
# sswap(total=17179865088, used=262144, free=17179602944, percent=0.0, sin=0, sout=0)
swap = psutil.swap_memory()
@ -84,6 +97,8 @@ def create_swap_data() -> Dict:
"used": swap.used,
"total": swap.total,
}
def create_drive_data() -> Dict:
tmp_elem = {}
# nvme0n1p1 => sdiskio(read_count=187, write_count=3, read_bytes=6002176, write_bytes=5120, read_time=36, write_time=7, read_merged_count=504, write_merged_count=0, busy_time=67)
@ -94,18 +109,19 @@ def create_drive_data() -> Dict:
# print(f"plop {mounting_points[elem]} ==> {drive_access.keys()}")
if mounting_points[elem] in drive_access:
tmp_elem[elem] = {
"read_bytes": drive_access[mounting_points[elem]].read_bytes,
"write_bytes": drive_access[mounting_points[elem]].write_bytes,
"used": drive_usage.used,
"total": drive_usage.total,
}
"read_bytes": drive_access[mounting_points[elem]].read_bytes,
"write_bytes": drive_access[mounting_points[elem]].write_bytes,
"used": drive_usage.used,
"total": drive_usage.total,
}
else:
tmp_elem[elem] = {
"used": drive_usage.used,
"total": drive_usage.total,
}
"used": drive_usage.used,
"total": drive_usage.total,
}
return tmp_elem
def create_network_data() -> Dict:
# eth0 => snetio(bytes_sent=0, bytes_recv=0, packets_sent=0, packets_recv=0, errin=0, errout=0, dropin=0, dropout=0)
data = psutil.net_io_counters(pernic=True)
@ -117,6 +133,7 @@ def create_network_data() -> Dict:
}
return tmp_elem
def agglutinate(configuration, name, data):
if configuration[name] == "auto":
return data
@ -125,10 +142,16 @@ def agglutinate(configuration, name, data):
return none
if __name__ == '__main__':
if __name__ == "__main__":
# Load arguments:
parser = argparse.ArgumentParser()
parser.add_argument("-c", "--config", type=str, default="/etc/karanage/system.json", help="json configuration file")
parser.add_argument(
"-c",
"--config",
type=str,
default="/etc/karanage/system.json",
help="json configuration file",
)
parser.add_argument("-C", "--connection", type=str, help="json configuration file")
# This element are read from the connection file:
@ -137,10 +160,16 @@ if __name__ == '__main__':
parser.add_argument("-T", "--token", type=str, help="Token to access to the server")
# This element are read from the configuration file:
parser.add_argument("-t", "--topic", type=str, default="PC/system", help="Topic of the message")
parser.add_argument("-s", "--sleep", type=int, default=30, help="Periodicity of the messages")
parser.add_argument("-d", "--display", help="Display the current state", action='store_true')
parser.add_argument(
"-t", "--topic", type=str, default="PC/system", help="Topic of the message"
)
parser.add_argument(
"-s", "--sleep", type=int, default=30, help="Periodicity of the messages"
)
parser.add_argument(
"-d", "--display", help="Display the current state", action="store_true"
)
args = parser.parse_args()
if Path(args.config).exists():
@ -164,12 +193,9 @@ if __name__ == '__main__':
configuration["config"]["sleep"] = args.sleep
if "topic" not in configuration["config"]:
configuration["config"]["topic"] = args.topic
connection = KaranageConnection(
url = args.url,
group = args.group,
token = args.token,
config_file = args.connection
url=args.url, group=args.group, token=args.token, config_file=args.connection
)
# create the rest interface of karanage
@ -201,7 +227,9 @@ if __name__ == '__main__':
print(json.dumps(out, indent=4))
# send message to the server:
try:
stateInterface.send(configuration["config"]["topic"], out, state = StateSystem.OK)
stateInterface.send(
configuration["config"]["topic"], out, state=StateSystem.OK
)
except KaranageException as ex:
print(f"Can not send to the server: {ex}")
time.sleep(configuration["config"]["sleep"])

36
client/python/karanage-tools/bin/karanage-tools-state-get → client/python/karanage_tools/karanage_tools/karanage_tools_state_get.py
View File

@ -1,11 +1,18 @@
#!/bin/python3
# Importing the library
#!/usr/bin/python3
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2023, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import argparse
from datetime import timezone
import json
from pathlib import Path
import subprocess
import time
from datetime import timezone
from typing import Dict, List
from karanage import (
@ -13,40 +20,37 @@ from karanage import (
KaranageException,
KaranageState,
)
import psutil
if __name__ == '__main__':
if __name__ == "__main__":
# Load arguments:
parser = argparse.ArgumentParser()
parser.add_argument("-C", "--connection", type=str, help="json configuration file")
parser.add_argument("-t", "--topic", type=str, help="Topic of the message")
parser.add_argument("-s", "--since", type=str, help="Iso date since the value time must be")
parser.add_argument("-w", "--watch", help="Display new states", action='store_true')
parser.add_argument(
"-s", "--since", type=str, help="Iso date since the value time must be"
)
parser.add_argument("-w", "--watch", help="Display new states", action="store_true")
# This element are read from the connection file:
parser.add_argument("-u", "--url", type=str, help="Base URL of the web service")
parser.add_argument("-g", "--group", type=str, help="Group the the message")
parser.add_argument("-T", "--token", type=str, help="Token to access to the server")
args = parser.parse_args()
connection = KaranageConnection(
url = args.url,
group = args.group,
token = args.token,
config_file = args.connection
url=args.url, group=args.group, token=args.token, config_file=args.connection
)
# create the rest interface of karanage
stateInterface = KaranageState(connection)
# transform since in a datetime:
while True:
data = stateInterface.gets(topic=args.topic, since=args.since)
for elem in data:
print(f"{elem.topic} @ {elem.time.astimezone()} => {elem.state}");
print(f"{elem.topic} @ {elem.time.astimezone()} => {elem.state}")
print(json.dumps(elem.data, indent=4))
args.since = elem.time
if not args.watch:

41
client/python/karanage-tools/bin/karanage-tools-state-history-get → client/python/karanage_tools/karanage_tools/karanage_tools_state_history_get.py
View File

@ -1,5 +1,12 @@
#!/bin/python3
# Importing the library
#!/usr/bin/python3
# -*- coding: utf-8 -*-
##
## @author Edouard DUPIN
##
## @copyright 2023, Edouard DUPIN, all right reserved
##
## @license MPL v2.0 (see license file)
##
import argparse
import json
from pathlib import Path
@ -8,37 +15,41 @@ import time
from typing import Dict, List
from karanage import KaranageConnection, KaranageState
import psutil
if __name__ == '__main__':
if __name__ == "__main__":
# Load arguments:
parser = argparse.ArgumentParser()
parser.add_argument("-C", "--connection", type=str, help="json configuration file")
parser.add_argument("-t", "--topic", type=str, help="Topic of the message")
parser.add_argument("-s", "--since", type=str, help="Iso date since the value time must be")
parser.add_argument("-S", "--since-id", type=str, help="Remote BDD id to start request")
parser.add_argument("-l", "--limit", type=int, default=100, help="Limit the number of request")
parser.add_argument(
"-s", "--since", type=str, help="Iso date since the value time must be"
)
parser.add_argument(
"-S", "--since-id", type=str, help="Remote BDD id to start request"
)
parser.add_argument(
"-l", "--limit", type=int, default=100, help="Limit the number of request"
)
# This element are read from the connection file:
parser.add_argument("-u", "--url", type=str, help="Base URL of the web service")
parser.add_argument("-g", "--group", type=str, help="Group the the message")
parser.add_argument("-T", "--token", type=str, help="Token to access to the server")
args = parser.parse_args()
connection = KaranageConnection(
url = args.url,
group = args.group,
token = args.token,
config_file = args.connection
url=args.url, group=args.group, token=args.token, config_file=args.connection
)
# create the rest interface of karanage
stateInterface = KaranageState(connection)
if args.topic is None:
print("Missing TOPIC ...")
else:
data = stateInterface.get_history(topic=args.topic, since=args.since, since_id=args.since_id, limit=args.limit)
print(f"Ret = {json.dumps(data, indent=4)}")
data = stateInterface.get_history(
topic=args.topic, since=args.since, since_id=args.since_id, limit=args.limit
)
print(f"Ret = {json.dumps(data, indent=4)}")

0
client/python/karanage-tools/setup.cfg → client/python/karanage_tools/setup.cfg
View File

4
client/python/karanage-tools/setup.py → client/python/karanage_tools/setup.py
View File

@ -30,7 +30,7 @@ def read_version_file():
# https://pypi.python.org/pypi?%3Aaction=list_classifiers
setup(
name="karanage-tools",
name="karanage_tools",
version=read_version_file(),
description="Karanage generic tools",
long_description=readme(),
@ -38,7 +38,7 @@ setup(
author="Edouard DUPIN",
author_email="yui.heero@gmail.com",
license="MPL-2",
packages=["karanage-tools"],
packages=["karanage_tools"],
classifiers=[
"Development Status :: 4 - Beta",
"License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",

0
client/python/karanage-tools/version.txt → client/python/karanage_tools/version.txt
View File