matt/Todoist-Actual-Backup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor API calls to respect rate limits

Browse Source
This commit is contained in:
Matt Wiseley
2025-10-18 11:00:45 -04:00
parent 31455f075e
commit 710e6749aa
1 changed files with 111 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

170
export_todoist.py
View File

@ -1,6 +1,8 @@
import os import os
import sys import sys
import json import json
import time
from collections import defaultdict
import requests import requests
from datetime import datetime, timedelta from datetime import datetime, timedelta
from todoist_api_python.api import TodoistAPI from todoist_api_python.api import TodoistAPI
@ -51,64 +53,108 @@ def download_attachment(url, filename):
return None return None
def fetch_all_projects(api): def _get_retry_delay(response, attempt, base_delay=5, max_delay=120):
ret = [] if response is not None:
projects_iter = api.get_projects() headers = getattr(response, "headers", {}) or {}
for projects in projects_iter: retry_after = headers.get("Retry-After") or headers.get("retry-after")
for project in projects: if retry_after:
name = getattr(project, 'name', None) try:
id = getattr(project, 'id', None) return max(1, int(float(retry_after)))
print(f"Found project {name} with ID {id}") except (TypeError, ValueError):
ret.append(project) pass
return ret reset_header = headers.get("X-RateLimit-Reset") or headers.get("x-rate-limit-reset")
if reset_header:
try:
reset_timestamp = float(reset_header)
return max(1, int(reset_timestamp - time.time()))
except (TypeError, ValueError):
pass
return min(max_delay, base_delay * (2 ** attempt))
def fetch_all_completed_tasks(api, project_id):
# Fetch all completed tasks for a project using get_completed_tasks_by_completion_date
# The API only allows up to 3 months per call, so we fetch just once for the last 3 months
all_completed = []
since = (datetime.now() - timedelta(days=90)).replace(hour=0, minute=0, second=0, microsecond=0)
until = datetime.now()
try:
completed_iter = api.get_completed_tasks_by_completion_date(since=since, until=until)
for completed_list in completed_iter:
for task in completed_list:
if hasattr(task, 'project_id') and str(task.project_id) == str(project_id):
all_completed.append(task)
except Exception as e:
print(f"Error fetching completed tasks for {since} to {until}: {e}")
print(f"Found {len(all_completed)} completed tasks for project {project_id}")
return all_completed
def fetch_all_tasks(api, project_id, completed=False): def execute_with_rate_limit(func, *args, **kwargs):
if completed: attempts = 0
return fetch_all_completed_tasks(api, project_id) max_attempts = 5
else: while True:
tasks = []
try: try:
tasks_iter = api.get_tasks(project_id=project_id) return func(*args, **kwargs)
for batch in tasks_iter: except Exception as error: # pylint: disable=broad-except
for task in batch: status_code = getattr(error, "status_code", None)
tasks.append(task) response = getattr(error, "response", None)
except Exception as e: if status_code is None and response is not None:
print(f"Error fetching active tasks for project {project_id}: {e}") status_code = getattr(response, "status_code", None)
print(f"Found {len(tasks)} completed tasks for project {project_id}") if status_code == 429 and attempts < max_attempts:
return tasks delay = _get_retry_delay(response, attempts)
attempts += 1
print(f"Rate limit hit for {func.__name__}. Waiting {delay} seconds before retry {attempts}/{max_attempts}...")
time.sleep(delay)
continue
raise
def fetch_comments(api, task_id): def fetch_all_projects(api):
comments = [] projects_by_id = {}
try: try:
comments_iter = api.get_comments(task_id=task_id) projects_iter = execute_with_rate_limit(api.get_projects)
for batch in comments_iter: for batch in projects_iter:
for comment in batch: for project in batch:
comments.append(comment) projects_by_id[str(getattr(project, "id", ""))] = project
except Exception: except Exception as error: # pylint: disable=broad-except
return [] print(f"Error fetching projects: {error}")
return comments return list(projects_by_id.values())
def process_task(api, task, completed=False): def fetch_active_tasks_by_project(api):
tasks_by_project = defaultdict(list)
try:
tasks_iter = execute_with_rate_limit(api.get_tasks)
for batch in tasks_iter:
for task in batch:
tasks_by_project[str(getattr(task, "project_id", ""))].append(task)
except Exception as error: # pylint: disable=broad-except
print(f"Error fetching active tasks: {error}")
print(f"Fetched active tasks for {len(tasks_by_project)} projects")
return tasks_by_project
def fetch_completed_tasks_by_project(api, since, until):
tasks_by_project = defaultdict(list)
try:
completed_iter = execute_with_rate_limit(
api.get_completed_tasks_by_completion_date,
since=since,
until=until,
)
for batch in completed_iter:
for task in batch:
tasks_by_project[str(getattr(task, "project_id", ""))].append(task)
except Exception as error: # pylint: disable=broad-except
print(f"Error fetching completed tasks between {since} and {until}: {error}")
print(f"Fetched completed tasks for {len(tasks_by_project)} projects")
return tasks_by_project
def fetch_comments_by_task(api, project_ids):
comments_by_task = defaultdict(list)
for project_id in project_ids:
try:
comments_iter = execute_with_rate_limit(api.get_comments, project_id=project_id)
for batch in comments_iter:
for comment in batch:
task_id = str(getattr(comment, "task_id", ""))
if task_id:
comments_by_task[task_id].append(comment)
except Exception as error: # pylint: disable=broad-except
print(f"Error fetching comments for project {project_id}: {error}")
print(f"Fetched comments for {len(comments_by_task)} tasks")
return comments_by_task
def process_task(task, comments_lookup):
task_dict = task.__dict__.copy() task_dict = task.__dict__.copy()
task_id = getattr(task, "id", None) or getattr(task, "task_id", None)
if task_id is not None:
task_dict.setdefault("id", task_id)
# Attachments (if any) # Attachments (if any)
attachments = [] attachments = []
if hasattr(task, 'attachments') and task.attachments: if hasattr(task, 'attachments') and task.attachments:
@ -123,9 +169,9 @@ def process_task(api, task, completed=False):
if attachments: if attachments:
task_dict['attachments'] = attachments task_dict['attachments'] = attachments
# Comments # Comments
comments = fetch_comments(api, task.id) comment_key = str(task_id) if task_id is not None else None
if comments: if comment_key and comment_key in comments_lookup:
task_dict['comments'] = [c.__dict__ for c in comments] task_dict['comments'] = [c.__dict__ for c in comments_lookup[comment_key]]
return task_dict return task_dict
@ -136,16 +182,22 @@ def main():
ensure_attachments_dir() ensure_attachments_dir()
api = TodoistAPI(get_api_key()) api = TodoistAPI(get_api_key())
projects = fetch_all_projects(api) projects = fetch_all_projects(api)
since = (datetime.now() - timedelta(days=90)).replace(hour=0, minute=0, second=0, microsecond=0)
until = datetime.now()
active_tasks_by_project = fetch_active_tasks_by_project(api)
completed_tasks_by_project = fetch_completed_tasks_by_project(api, since=since, until=until)
comment_project_ids = sorted(
pid for pid in (set(active_tasks_by_project.keys()) | set(completed_tasks_by_project.keys())) if pid
)
comments_by_task = fetch_comments_by_task(api, comment_project_ids)
data = [] data = []
for project in projects: for project in projects:
project_dict = project.__dict__.copy() project_dict = project.__dict__.copy()
project_id = project.id project_id = str(getattr(project, "id", ""))
# Active tasks active_tasks = active_tasks_by_project.get(project_id, [])
active_tasks = fetch_all_tasks(api, project_id, completed=False) completed_tasks = completed_tasks_by_project.get(project_id, [])
# Completed tasks project_dict['tasks'] = [process_task(t, comments_by_task) for t in active_tasks]
completed_tasks = fetch_all_tasks(api, project_id, completed=True) project_dict['completed_tasks'] = [process_task(t, comments_by_task) for t in completed_tasks]
project_dict['tasks'] = [process_task(api, t, completed=False) for t in active_tasks]
project_dict['completed_tasks'] = [process_task(api, t, completed=True) for t in completed_tasks]
data.append(project_dict) data.append(project_dict)
# Write JSON # Write JSON
today = datetime.now().strftime("%Y-%m-%d") today = datetime.now().strftime("%Y-%m-%d")