Claude/document api mk12resenp98brab e pm yp

api/__init__.py

@@ -1,6 +1,6 @@
 from flask import Flask, current_app, render_template, request, session
 from flask_cors import CORS
-from .modules import tweets, log
+from .modules import toots, log
 from .modules.auth import login_exempt
 from lib.shared import authToken, secretKey
 from datetime import timedelta
@@ -12,7 +12,7 @@ def create_app(test_config = None):
 
     cors = CORS(app)
 
-    app.register_blueprint(tweets.bp)
+    app.register_blueprint(toots.bp)
     app.register_blueprint(log.bp)
 
     @app.route('/')

api/modules/toots.py

@@ -0,0 +1,148 @@
+from datetime import datetime
+import logging
+import json
+import os
+
+from flask import Blueprint, jsonify, Response, render_template, abort
+from flask_cors import cross_origin
+from .auth import login_exempt
+
+from lib.mastodon_base import readTootsApiJson
+from lib.Toot import Toot
+from lib.TootForest import TootForest
+from lib.shared import data_folder
+
+bp = Blueprint('tweets', __name__, url_prefix='/tweets')
+
+@bp.route('/')
+@cross_origin()
+@login_exempt
+def root():
+    # Read the current tweets from the API or other source
+    current_tweets = readTootsApiJson()
+
+    # Read additional tweets from file_tweets_transformed.json
+    try:
+        with open(os.path.join(data_folder, 'file_tweets_transformed.json'), 'r', encoding='utf-8') as file:
+            file_tweets_data = json.load(file)
+            # Check if the data is a dictionary
+            if isinstance(file_tweets_data, dict):
+                # If the current tweets are also a dictionary, update it
+                if isinstance(current_tweets, dict):
+                    current_tweets.update(file_tweets_data)
+                # If the current tweets are a list, convert the dictionary to a list and extend it
+                elif isinstance(current_tweets, list):
+                    current_tweets.extend(file_tweets_data.values())
+                else:
+                    raise ValueError("The current tweets are neither a list nor a dictionary.")
+            else:
+                raise ValueError("The contents of file_tweets_transformed.json are not a dictionary")
+    except FileNotFoundError:
+        logging.error("The file file_tweets_transformed.json was not found.")
+    except json.JSONDecodeError:
+        logging.error("The file file_tweets_transformed.json does not contain valid JSON.")
+    except ValueError as e:
+        logging.error(e)
+
+    # Return the combined data as a JSON response
+    return Response(json.dumps({
+        'date': int(datetime.timestamp(datetime.now())),
+        'tweets': current_tweets
+    }), mimetype='application/json')
+
+
+@bp.route('/<int:id>')
+def tweet(id):
+    tweet = Toot.loadFromFile(id)
+    return jsonify(tweet.data)
+
+@bp.route('/forest')
+def forest_show():
+    return render_template('forest.html.j2', forest = TootForest.fromFolder())
+    #return Response(str(forest), mimetype='text/plain')
+
+@bp.route('/forest/renew')
+def forest_create():
+    logging.warning('Manual invocation of creating forest!')
+    forest = TootForest.fromFolder()
+    forest.saveApiJson()
+
+    # Read the current tweets from the API or other source
+    current_tweets = readTootsApiJson()
+
+    # Read additional tweets from file_tweets_transformed.json
+    try:
+        with open(os.path.join(data_folder, 'file_tweets_transformed.json'), 'r', encoding='utf-8') as file:
+            file_tweets_data = json.load(file)
+            # Check if the data is a dictionary
+            if isinstance(file_tweets_data, dict):
+                # If the current tweets are also a dictionary, update it
+                if isinstance(current_tweets, dict):
+                    current_tweets.update(file_tweets_data)
+                # If the current tweets are a list, convert the dictionary to a list and extend it
+                elif isinstance(current_tweets, list):
+                    current_tweets.extend(file_tweets_data.values())
+                else:
+                    raise ValueError("The current tweets are neither a list nor a dictionary.")
+            else:
+                raise ValueError("The contents of file_tweets_transformed.json are not a dictionary")
+    except FileNotFoundError:
+        logging.error("The file file_tweets_transformed.json was not found.")
+    except json.JSONDecodeError:
+        logging.error("The file file_tweets_transformed.json does not contain valid JSON.")
+    except ValueError as e:
+        logging.error(e)
+
+    # Return the combined data as a JSON response
+    return Response(json.dumps(current_tweets), mimetype='application/json')
+
+@bp.route('/add/<int:id>')
+def add(id):
+    logging.warning('Manual invocation of adding tweet (id: {})!'.format(id))
+    tweet = Toot.loadFromMastodon(id)
+    tweet.save()
+    # renew forest
+    forest = TootForest.fromFolder()
+    forest.saveApiJson()
+    return jsonify({
+        'message': 'added',
+        'tweet': {
+            'id': id,
+            'data': tweet.data
+        }
+    })
+
+@bp.route('/delete/<int:id>')
+def delete(id):
+    logging.warning('Manual invocation of deleting tweet (id: {})!'.format(id))
+    tweet = Toot.loadFromFile(id)
+    tweet.delete()
+    # renew forest
+    forest = TootForest.fromFolder()
+    forest.saveApiJson()
+    return jsonify({
+        'message': 'deleted',
+        'tweet': {
+            'id': id,
+            'data': tweet.data
+        }
+    })
+
+@bp.route('/all')
+def all():
+    tweets = Toot.loadFromFolder()
+    tweets.sort(key = lambda x: x.getDateTime(), reverse = True)
+    # [Toot(i) for i in tweets]
+    return render_template('all.html.j2', tweets = tweets)
+
+@bp.route('/stories')
+def info():
+    tweets = readTootsApiJson()
+    stories = {};
+    for id, info in tweets.items():
+        if 'story' in info:
+            storyId = info['story']
+            if storyId not in stories:
+                stories[storyId] = []
+            stories[storyId].append(Toot.loadFromFile(id))
+    return render_template('stories.html.j2', stories = stories)

config.template.json

@@ -1,6 +1,16 @@
 {
     "authToken": "xxx",
     "secretKey": "xxx",
+    "mastodon": {
+        "instance_url": "https://mastodon.instance",
+        "client": {
+            "key": "xxx",
+            "secret": "xxx"
+        },
+        "access": {
+            "token": "xxx"
+        }
+    },
     "twitter": {
         "api": {
             "key": "xxx",

downloadFromApi.py

@@ -0,0 +1,43 @@
+import os
+import requests
+import json
+from lib.shared import authToken
+
+BASE_URL = "https://dev.decarbnow.space"
+AUTH_URL = BASE_URL + "/authenticate/"
+DOWNLOAD_URL = BASE_URL + "/tweets/"
+
+# Create a session to maintain authentication
+session = requests.Session()
+
+# Authenticate using the provided token
+auth_response = session.get(AUTH_URL + authToken)
+if auth_response.status_code != 200:
+    print(f"Authentication failed with status code: {auth_response.status_code}")
+    exit()
+
+# Path to the data folder
+data_folder = "data"
+os.makedirs(data_folder, exist_ok=True)
+
+# Load tweet IDs from fromAPI.json
+with open(os.path.join(data_folder, "fromAPI.json"), "r", encoding='utf-8') as json_file:
+    tweet_data = json.load(json_file)
+    tweet_ids = tweet_data.get("tweets", {}).keys()
+
+# Download files for each tweet ID
+for tweet_id in tweet_ids:
+    download_url = DOWNLOAD_URL + tweet_id
+
+    # Send request to download the file
+    file_response = session.get(download_url)
+    if file_response.status_code == 200:
+        # Save the file in the data folder
+        with open(os.path.join(data_folder, "live", f"{tweet_id}.json"), "wb") as file:
+            file.write(file_response.content)
+        print(f"Downloaded file for tweet ID {tweet_id}")
+    else:
+        print(f"Failed to download file for tweet ID {tweet_id} with status code: {file_response.status_code}")
+
+# Close the session
+session.close()

getOldToots.py

@@ -0,0 +1,61 @@
+from mastodon import Mastodon
+import json
+from datetime import datetime
+import os
+import shutil
+from lib.mastodon_auth import get_auth_user
+
+from lib.shared import base_folder, tootsFetchSettings, toots_folder
+
+# Define the URL you want to check for in the messages
+keyword = tootsFetchSettings['listen'] + '/@'
+
+# Define the path for the data folder and archive subfolder
+data_folder = os.path.join(toots_folder, tootsFetchSettings['folder'])
+
+archive_folder = os.path.join(data_folder, "archive")
+
+# Ensure the data and archive folders exist
+os.makedirs(data_folder, exist_ok=True)
+os.makedirs(archive_folder, exist_ok=True)
+
+# Get a list of existing message IDs in the data folder
+existing_ids = [filename.split(".")[0] for filename in os.listdir(data_folder) if filename.endswith(".json")]
+
+# Create an instance of Mastodon
+mastodon = get_auth_user()
+
+# Custom JSON encoder to handle datetime objects
+class DateTimeEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, datetime):
+            return obj.strftime('%Y-%m-%d %H:%M:%S')
+        return super().default(obj)
+
+# Search for messages containing the target URL
+search_results = mastodon.search(keyword)
+
+# Check each search result for the target URL
+for status in search_results['statuses']:
+    # Generate a unique filename using ID
+    message_id = str(status['id'])
+    filename = f'{message_id}.json'
+    filepath = os.path.join(data_folder, filename)
+
+    # Save the relevant information to a JSON file
+    with open(filepath, 'w') as output_file:
+        json.dump(status, output_file, cls=DateTimeEncoder)
+
+    print(f'Message with ID {message_id} saved to {filepath}')
+
+    # Remove the ID from the existing IDs list if found
+    if message_id in existing_ids:
+        existing_ids.remove(message_id)
+
+# Move any remaining files in the data folder to the archive folder
+for message_id in existing_ids:
+    filename = f'{message_id}.json'
+    filepath = os.path.join(data_folder, filename)
+    archive_filepath = os.path.join(archive_folder, filename)
+    shutil.move(filepath, archive_filepath)
+    print(f'Message with ID {message_id} moved to archive folder')