From e6bf16a74cf44ba3a5355b76ec7f8ae2cb3a6844 Mon Sep 17 00:00:00 2001 From: Peter Neuer Date: Tue, 20 Aug 2024 21:24:02 +0000 Subject: [PATCH] improvement --- scripts/diagnose_file_log.py | 116 ++++++++++++ scripts/fhem_fetch.py | 166 ++++++++++++++---- scripts/fhem_fetch_gradio.py | 94 ++++++++++ scripts/init.py | 3 +- scripts/read_devices.py | 116 ++++++++++++ scripts/test_database.py | 38 ---- utilities/__init__.py | 2 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 186 bytes .../__pycache__/get_token.cpython-310.pyc | Bin 0 -> 1215 bytes utilities/get_token.py | 43 +++++ 10 files changed, 509 insertions(+), 69 deletions(-) create mode 100644 scripts/diagnose_file_log.py create mode 100644 scripts/fhem_fetch_gradio.py create mode 100644 scripts/read_devices.py delete mode 100644 scripts/test_database.py create mode 100644 utilities/__init__.py create mode 100644 utilities/__pycache__/__init__.cpython-310.pyc create mode 100644 utilities/__pycache__/get_token.cpython-310.pyc create mode 100644 utilities/get_token.py diff --git a/scripts/diagnose_file_log.py b/scripts/diagnose_file_log.py new file mode 100644 index 0000000..c72fe9a --- /dev/null +++ b/scripts/diagnose_file_log.py @@ -0,0 +1,116 @@ +import re +import requests +from bs4 import BeautifulSoup +from requests.exceptions import HTTPError, Timeout, RequestException + +# FHEM server URL base +FHEM_URL_BASE = "https://fhem.auwiesen2.de/fhem" + +# CSRF token (replace with your actual token) +CSRF_TOKEN = "csrf_611440676390392" + +# Headers including CSRF token +HEADERS = { + "X-FHEM-csrfToken": CSRF_TOKEN, + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "text/html" +} + +# Session to handle requests +session = requests.Session() + +def fetch_device_log(device_id, year): + """Fetch the log file for the specified device and year.""" + log_url = f"{FHEM_URL_BASE}/FileLog_logWrapper&dev=FileLog_{device_id}&type=text&file={device_id}-{year}.log" + + try: + response = session.get(log_url, headers=HEADERS, timeout=10) + response.raise_for_status() + soup = BeautifulSoup(response.text, 'html.parser') + + # Extract the log data + log_content = soup.find('pre', class_='log') + if log_content: + log_text = log_content.text.strip() + # Only keep lines that start with the expected date format + log_lines = [line for line in log_text.splitlines() if re.match(r'^\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}', line)] + return log_lines + else: + print(f"No log data found for device {device_id} in year {year}.") + return None + + except (HTTPError, Timeout) as err: + print(f"Error fetching log for device {device_id}: {err}") + return None + except RequestException as req_err: + print(f"An error occurred: {req_err}") + return None + +def analyze_log_format(log_lines): + """Analyze the log lines to propose a table structure.""" + if not log_lines: + return None + + parameter_patterns = {} + timestamp_pattern = r"(\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2})" + device_pattern = r"(MA_[a-zA-Z0-9]+)" + + for line in log_lines: + match = re.match(fr"{timestamp_pattern} {device_pattern} (.*)", line) + if match: + timestamp = match.group(1) + device_id = match.group(2) + parameters = match.group(3).split() + + for param in parameters: + key_value = param.split(":") + if len(key_value) == 2: + key, value = key_value[0], key_value[1] + if key not in parameter_patterns: + if value.isdigit(): + parameter_patterns[key] = "INT" + elif re.match(r"^\d+(\.\d+)?$", value): + parameter_patterns[key] = "FLOAT" + else: + parameter_patterns[key] = "VARCHAR(255)" + + # Propose table structure + table_structure = { + "device_id": "VARCHAR(255)", + "timestamp": "DATETIME" + } + table_structure.update(parameter_patterns) + + return table_structure + +def print_table_structure(table_structure): + """Print the proposed table structure.""" + print("Proposed Table Structure:") + print("CREATE TABLE device_logs (") + for column, dtype in table_structure.items(): + print(f" {column} {dtype},") + print(" PRIMARY KEY (device_id, timestamp)") + print(");") + +def main(device_id, year=2024): + log_lines = fetch_device_log(device_id, year) + + if log_lines: + # Print the first 100 lines of the log + print(f"First 100 lines of the log for device {device_id}:\n") + for i, line in enumerate(log_lines[:100]): + print(f"{i + 1}: {line}") + + table_structure = analyze_log_format(log_lines) + if table_structure: + print("\nAnalyzing log format...") + print_table_structure(table_structure) + else: + print("No valid log data found to analyze.") + else: + print("No log data retrieved.") + +if __name__ == "__main__": + # Example device ID to test with + example_device_id = "MA_030e8b3e5bc3" + main(example_device_id) diff --git a/scripts/fhem_fetch.py b/scripts/fhem_fetch.py index 39ad47b..6a286b3 100644 --- a/scripts/fhem_fetch.py +++ b/scripts/fhem_fetch.py @@ -1,37 +1,143 @@ +import mysql.connector +from mysql.connector import Error +import os +import re import requests +from bs4 import BeautifulSoup +from requests.exceptions import HTTPError, Timeout, RequestException -# FHEM URL -# FHEM_URL = "http://192.168.2.200:8083/fhem" -FHEM_URL = "https://fhem.auwiesen2.de/fhem" +# Import the get_token function from the utilities module +from utilities.get_token import get_token -# Parameters including CSRF token -PARAMS = { - "cmd": "jsonlist2", - "XHR": "1", - "fwcsrf": "csrf_334814865250639" # CSRF token as a parameter -} +# Database connection details +DB_HOST = 'mariadb' +DB_USER = 'root' +DB_PASSWORD = os.getenv('MYSQL_ROOT_PASSWORD', 'S3raph1n!') +DB_NAME = 'fhem' +TABLE_NAME = 'devices' -# Headers including CSRF token -HEADERS = { - "X-FHEM-csrfToken": "csrf_822558611144652", - "Content-Type": "application/x-www-form-urlencoded", - "Accept": "application/json" -} +# FHEM server URL +FHEM_URL_BASE = "https://fhem.auwiesen2.de/fhem" -# Send the request with both parameters and headers -response = requests.get(FHEM_URL, params=PARAMS, headers=HEADERS) +# Path to the log file created by the first script +LOG_FILE_PATH = 'fhem_script.log' -# Debugging: Check the status code and response content -print("Response Status Code:", response.status_code) -print("Response Content:", response.text) +# Session to handle requests +session = requests.Session() -try: - # Attempt to parse the response as JSON - data = response.json() - devices = data['Results'] - for device in devices: - print(f"Device: {device['Name']}, State: {device['Internals']['STATE']}") -except requests.exceptions.JSONDecodeError: - print("Error: Failed to decode JSON response.") -except KeyError: - print("Error: Expected keys not found in the JSON data.") +def connect_to_db(): + """Establish a connection to the MySQL database.""" + connection = None + try: + connection = mysql.connector.connect( + host=DB_HOST, + user=DB_USER, + password=DB_PASSWORD, + charset='utf8mb4', + collation='utf8mb4_unicode_ci' + ) + if connection.is_connected(): + print(f"Connected to MySQL Server.") + return connection + except Error as e: + print(f"Error while connecting to MySQL: {e}") + return connection + +def create_database(connection): + """Create the fhem database if it doesn't exist.""" + try: + cursor = connection.cursor() + cursor.execute(f"CREATE DATABASE IF NOT EXISTS {DB_NAME} CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;") + print(f"Database '{DB_NAME}' created or already exists with utf8mb4_unicode_ci collation.") + except Error as e: + print(f"Error creating database: {e}") + +def create_table(connection): + """Create the devices table in the fhem database.""" + try: + cursor = connection.cursor() + cursor.execute(f"USE {DB_NAME};") + create_table_query = f""" + CREATE TABLE IF NOT EXISTS {TABLE_NAME} ( + Device_id VARCHAR(255) PRIMARY KEY, + Room VARCHAR(255), + alias VARCHAR(255) NOT NULL + ) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + """ + cursor.execute(create_table_query) + print(f"Table '{TABLE_NAME}' created or already exists in database '{DB_NAME}'.") + except Error as e: + print(f"Error creating table: {e}") + +def insert_device_data(connection, device_id, room, alias): + """Insert a device's data into the devices table.""" + try: + cursor = connection.cursor() + insert_query = f""" + INSERT INTO {TABLE_NAME} (Device_id, Room, alias) + VALUES (%s, %s, %s) + ON DUPLICATE KEY UPDATE Room = VALUES(Room), alias = VALUES(alias); + """ + cursor.execute(insert_query, (device_id, room, alias)) + connection.commit() + print(f"Inserted/Updated Device: {device_id}, Room: {room}, Alias: {alias}") + except Error as e: + print(f"Error inserting data: {e}") + +def get_device_details(device_name, csrf_token): + """Fetch the room and alias attributes for a given device from FHEM.""" + try: + detail_url = f"{FHEM_URL_BASE}?detail={device_name}" + headers = { + "X-FHEM-csrfToken": csrf_token, + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "application/json" + } + response = session.get(detail_url, headers=headers, timeout=10) + response.raise_for_status() + + soup = BeautifulSoup(response.text, 'html.parser') + attributes_table = soup.find('table', class_='block wide attributes') + + alias = None + room = None + if attributes_table: + rows = attributes_table.find_all('tr') + for row in rows: + name_div = row.find('div', class_='dname') + value_div = row.find('div', class_='dval') + if name_div and value_div: + attr_name = name_div.text.strip() + attr_value = value_div.text.strip() + if attr_name == "alias": + alias = attr_value + elif attr_name == "room": + room = attr_value + + return alias, room + + except (HTTPError, Timeout) as err: + print(f"Error fetching details for device {device_name}: {err}") + return None, None + except RequestException as req_err: + print(f"An error occurred: {req_err}") + return None, None + +def read_log_and_insert_into_db(connection, csrf_token): + """Read the log file, fetch device details, and insert them into the database.""" + try: + with open(LOG_FILE_PATH, 'r') as log_file: + for line in log_file: + match = re.search(r'Device: (MA_[a-zA-Z0-9]+), State:', line) + if match: + device_name = match.group(1) + alias, room = get_device_details(device_name, csrf_token) + if alias and room: + insert_device_data(connection, device_name, room, alias) + elif alias: # In case room is not defined + insert_device_data(connection, device_name, None, alias) + + except FileNotFoundError: + print(f"Log file '{LOG_FILE_PATH}' not found.") + +def main(): diff --git a/scripts/fhem_fetch_gradio.py b/scripts/fhem_fetch_gradio.py new file mode 100644 index 0000000..418c659 --- /dev/null +++ b/scripts/fhem_fetch_gradio.py @@ -0,0 +1,94 @@ +import os +import requests +import gradio as gr +import subprocess + +# FHEM URL +FHEM_URL = "https://fhem.auwiesen2.de/fhem" + +# Parameters including CSRF token +PARAMS = { + "cmd": "jsonlist2", + "XHR": "1", + "fwcsrf": "csrf_611440676390392" # CSRF token as a parameter +} + +# Headers including CSRF token +HEADERS = { + "X-FHEM-csrfToken": "csrf_611440676390392", + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "application/json" +} + +def get_device_list(): + try: + # Send the request with both parameters and headers + response = requests.get(FHEM_URL, params=PARAMS, headers=HEADERS, timeout=10) + + # Debugging: Check the status code and response content + print("Response Status Code:", response.status_code) + print("Response Content:", response.text) + + response.raise_for_status() # Raise an HTTPError for bad responses (4xx and 5xx) + + # Attempt to parse the response as JSON + data = response.json() + print("Parsed JSON Data:", data) # Debugging: Print the parsed JSON data + + devices = data.get('Results', []) + + # Prepare data for Gradio (convert dictionaries to tuples) + device_list = [] + for device in devices: + device_tuple = (device['Name'], device['Internals'].get('STATE', 'Unknown')) + device_list.append(device_tuple) + + return device_list + except requests.exceptions.JSONDecodeError: + return "Error: Failed to decode JSON response." + except KeyError: + return "Error: Expected keys not found in the JSON data." + except requests.exceptions.RequestException as e: + return f"Error: {str(e)}" + +def display_devices(): + device_list = get_device_list() + if isinstance(device_list, str): # Check if an error message was returned + return device_list + else: + # Return the device list as a Gradio Dataframe component + return gr.Dataframe(device_list, headers=["Device", "State"]) + +# Create a Gradio interface +interface = gr.Interface(fn=display_devices, inputs=[], outputs=gr.Dataframe(headers=["Device", "State"])) + +# List of ports to try +ports_to_try = [8081, 8082] + +# Function to terminate processes using a specific port +def kill_process_on_port(port): + try: + pid = subprocess.check_output(f"lsof -t -i:{port}", shell=True).decode().strip() + if pid: + os.system(f"kill {pid}") + print(f"Terminated process {pid} on port {port}") + except subprocess.CalledProcessError: + print(f"No process found on port {port}") + +# Terminate any process that might be using the desired ports +for port in ports_to_try: + kill_process_on_port(port) + +# Try each port until one is available +for port in ports_to_try: + try: + # Launch the interface with the specified port + interface.launch( + server_name="0.0.0.0", # Bind to all IPs inside the Docker container + server_port=port, # Try the current port + share=False, # Do not create a public link since you use a custom domain + inbrowser=False # Do not automatically open a browser + ) + break # If launch is successful, break out of the loop + except OSError: + print(f"Port {port} is not available. Trying next port...") diff --git a/scripts/init.py b/scripts/init.py index 35e6b23..d23c1c2 100644 --- a/scripts/init.py +++ b/scripts/init.py @@ -1 +1,2 @@ -!pip install mysql-connector-python +pip install mysql-connector-python +pip install gradio diff --git a/scripts/read_devices.py b/scripts/read_devices.py new file mode 100644 index 0000000..5144022 --- /dev/null +++ b/scripts/read_devices.py @@ -0,0 +1,116 @@ +import re +import requests +import sys +import os +from bs4 import BeautifulSoup +from requests.exceptions import HTTPError, Timeout, RequestException + +# Set the working directory to parent +os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +# Update sys.path to include the parent directory of the current script +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +# Import the get_token function +from utilities.get_token import get_token + +# FHEM URL base (replace with your actual FHEM server URL) +FHEM_URL_BASE = "https://fhem.auwiesen2.de/fhem" + +# Retrieve the CSRF token dynamically +csrf_token = get_token(FHEM_URL_BASE) + +# Headers including CSRF token +HEADERS = { + "X-FHEM-csrfToken": csrf_token, + "Content-Type": "application/x-www-form-urlencoded", + "Accept": "application/json" +} + +# Path to the log file created by the first script +log_file_path = 'fhem_script.log' + +# Define the pattern to match devices in the log file +device_pattern = r'^attr (MA_[a-zA-Z0-9]+) alias (.+)$' + +# Session to handle requests +session = requests.Session() + +def get_device_attributes(device_name): + """Fetch the room and alias attributes for a given device from FHEM.""" + try: + # Construct the URL to get the device details + detail_url = f"{FHEM_URL_BASE}?detail={device_name}" + + # Send the request + response = session.get(detail_url, headers=HEADERS, timeout=10) + response.raise_for_status() + + # Parse the HTML response using BeautifulSoup + soup = BeautifulSoup(response.text, 'html.parser') + + # Find the table with the class 'block wide attributes' + attributes_table = soup.find('table', class_='block wide attributes') + + # Extract the alias and room values + alias = None + room = None + if attributes_table: + rows = attributes_table.find_all('tr') + for row in rows: + name_div = row.find('div', class_='dname') + value_div = row.find('div', class_='dval') + if name_div and value_div: + attr_name = name_div.text.strip() + attr_value = value_div.text.strip() + if attr_name == "alias": + alias = attr_value + elif attr_name == "room": + room = attr_value + + return alias, room + + except (HTTPError, Timeout) as err: + print(f"Error fetching attributes for device {device_name}: {err}") + return None, None + except RequestException as req_err: + print(f"An error occurred: {req_err}") + return None, None + +def list_ma_devices_with_aliases_and_rooms(log_file_path, device_pattern): + matched_devices = [] + + try: + # Open the log file and read line by line + with open(log_file_path, 'r') as log_file: + for line in log_file: + # Check if the line matches the pattern for devices starting with "MA_" + match = re.search(device_pattern, line) + if match: + device_name = match.group(1) + + # Get the alias and room for the matched device + alias, room = get_device_attributes(device_name) + + # Add the device name, alias, and room to the list + matched_devices.append((device_name, alias, room)) + + except FileNotFoundError: + print(f"Log file '{log_file_path}' not found.") + return None + + return matched_devices + +def main(): + # List all devices starting with "MA_", their aliases, and rooms + devices = list_ma_devices_with_aliases_and_rooms(log_file_path, device_pattern) + + if devices: + print("Devices starting with 'MA_', their aliases, and rooms:") + for device_name, alias, room in devices: + print(f"Device: {device_name}, Alias: {alias}, Room: {room}") + else: + print("No devices starting with 'MA_' found.") + +if __name__ == "__main__": + main() diff --git a/scripts/test_database.py b/scripts/test_database.py deleted file mode 100644 index b2a47c0..0000000 --- a/scripts/test_database.py +++ /dev/null @@ -1,38 +0,0 @@ -import mysql.connector -from mysql.connector import errorcode - -def connect_to_db(): - connection = None # Initialize connection to None - try: - # Establish connection - connection = mysql.connector.connect( - host='mariadb', # Use the container name or the IP address of the host running the container - user='neuer', - password='S3raph1n!', - charset='utf8mb4' # Ensure the connection uses the correct charset - ) - - # Create a cursor object using the connection - cursor = connection.cursor() - - # Set the session collation explicitly - cursor.execute("SET SESSION collation_connection = 'utf8mb4_unicode_ci'") - cursor.execute("SET NAMES 'utf8mb4' COLLATE 'utf8mb4_unicode_ci'") - - print("Connected to MariaDB with utf8mb4_unicode_ci collation") - return connection - - except mysql.connector.Error as err: - if err.errno == errorcode.ER_ACCESS_DENIED_ERROR: - print("Something is wrong with your user name or password") - elif err.errno == errorcode.ER_BAD_DB_ERROR: - print("Database does not exist") - else: - print(err) - return connection - -# Example of using the connection -conn = connect_to_db() -if conn: - # Perform database operations here - conn.close() diff --git a/utilities/__init__.py b/utilities/__init__.py new file mode 100644 index 0000000..6c30ee9 --- /dev/null +++ b/utilities/__init__.py @@ -0,0 +1,2 @@ +# db/__init__.py +print("Initializing the db package") diff --git a/utilities/__pycache__/__init__.cpython-310.pyc b/utilities/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8a0e78b50ba191ca03752bd3f75b591fcec99cc9 GIT binary patch literal 186 zcmd1j<>g`kg6^h6X=XtBF^Gc17VF2yX97)+kJl@xyv1RYo1apelWNBZatX*Z G9!3BS#4t(# literal 0 HcmV?d00001 diff --git a/utilities/__pycache__/get_token.cpython-310.pyc b/utilities/__pycache__/get_token.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..01b8932e6b6dca3f0951c1c80632eda87b37047a GIT binary patch literal 1215 zcmYjQTaVL56rLI19Ves#RH)RFN-ZUoNXklxYE^FAZp8ydh%7|p#WI=6#*1T{nK9`` z4nkV`i0C^)N*?<++J7;xec~@H65>oQD`qsG&;1;K_GHznk6^w2_rKxa7DE4P<@05L zatn@l1V9nR3F@z&cue{RUdiUz>f3AC={uD4UFK5bOVszMNv&6??=zoyN4Q^M0kv7> zEqX_eFl68^YB}$rHo7;xFp@GC+nvsE#K!HYn8Zx5bgNC-(sFa^QzoN0x!&8|8DO2W zUiuaE`4x_6f=8k`F3|$daETAdi~t%lqeOGFByXWCU|cnwh0J_bEh_q%;;LMqsa$CRDaS3Q7wk8>DJ!FW_~i*Q9$;D>W%LbzU29zMDCv zGe5OJ^E!1Q)-7Srzo6a%Q}h=(kF*8qYn#8o8%qxnsA0lB0*ZU6i>@G%OrV3_kKy#d z-vvMPYaG78Xy!rpQ}Q&el-|La4ZS^xrQmzv(V)AQCU}Bc)qBTy`xxI+Na5zx_KQ6`OKq#PNKSysrYRmkDxpOrNbd>9_%AEy4p2X}Wl&-ixp_tshE7Cgx#F4$5M zc@j&I1mkHm23sBpG08Yp4rh514VW^7NYq#1fU`)lut=XX9uH$i!$Fox_NP=|Z+w&u zqau;v@~=2?JJ2wRpKPfMclP$~-(T1NW1sh8Ir?pt@_?d;s{?gC}JsZF{*zo_9G4f