64 lines
No EOL
1.8 KiB
Python
64 lines
No EOL
1.8 KiB
Python
import os
|
|
import json
|
|
import markovify
|
|
import pickle
|
|
from modules.globalvars import *
|
|
from modules.translations import *
|
|
def get_file_info(file_path):
|
|
try:
|
|
file_size = os.path.getsize(file_path)
|
|
with open(file_path, "r") as f:
|
|
lines = f.readlines()
|
|
return {"file_size_bytes": file_size, "line_count": len(lines)}
|
|
except Exception as e:
|
|
return {"error": str(e)}
|
|
|
|
def load_memory():
|
|
data = []
|
|
|
|
# load data from MEMORY_FILE
|
|
try:
|
|
with open(MEMORY_FILE, "r") as f:
|
|
data = json.load(f)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
if not os.path.exists(MEMORY_LOADED_FILE):
|
|
try:
|
|
with open(DEFAULT_DATASET_FILE, "r") as f:
|
|
default_data = json.load(f)
|
|
data.extend(default_data)
|
|
except FileNotFoundError:
|
|
pass
|
|
with open(MEMORY_LOADED_FILE, "w") as f:
|
|
f.write("Data loaded")
|
|
return data
|
|
|
|
def save_memory(memory):
|
|
with open(MEMORY_FILE, "w") as f:
|
|
json.dump(memory, f, indent=4)
|
|
|
|
def train_markov_model(memory, additional_data=None):
|
|
if not memory:
|
|
return None
|
|
text = "\n".join(memory)
|
|
if additional_data:
|
|
text += "\n" + "\n".join(additional_data)
|
|
model = markovify.NewlineText(text, state_size=2)
|
|
return model
|
|
|
|
def save_markov_model(model, filename='markov_model.pkl'):
|
|
with open(filename, 'wb') as f:
|
|
pickle.dump(model, f)
|
|
print(f"Markov model saved to {filename}.")
|
|
|
|
def load_markov_model(filename='markov_model.pkl'):
|
|
|
|
try:
|
|
with open(filename, 'rb') as f:
|
|
model = pickle.load(f)
|
|
print(f"{GREEN}{get_translation(LOCALE, 'model_loaded')} {filename}.{RESET}")
|
|
return model
|
|
except FileNotFoundError:
|
|
print(f"{RED}{filename} {get_translation(LOCALE, 'not_found')}{RESET}")
|
|
return None |