matt/Todoist-Actual-Backup
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor API calls to respect rate limits

Browse Source
This commit is contained in:
Matt Wiseley
2025-10-18 11:00:45 -04:00
parent 31455f075e
commit 710e6749aa
1 changed files with 111 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

160
export_todoist.py
View File

@ -1,6 +1,8 @@
import os import os
import sys import sys
import json import json
import time
from collections import defaultdict
import requests import requests
from datetime import datetime, timedelta from datetime import datetime, timedelta
from todoist_api_python.api import TodoistAPI from todoist_api_python.api import TodoistAPI
@ -51,64 +53,108 @@ def download_attachment(url, filename):
return None return None
def _get_retry_delay(response, attempt, base_delay=5, max_delay=120):
if response is not None:
headers = getattr(response, "headers", {}) or {}
retry_after = headers.get("Retry-After") or headers.get("retry-after")
if retry_after:
try:
return max(1, int(float(retry_after)))
except (TypeError, ValueError):
pass
reset_header = headers.get("X-RateLimit-Reset") or headers.get("x-rate-limit-reset")
if reset_header:
try:
reset_timestamp = float(reset_header)
return max(1, int(reset_timestamp - time.time()))
except (TypeError, ValueError):
pass
return min(max_delay, base_delay * (2 ** attempt))
def execute_with_rate_limit(func, *args, **kwargs):
attempts = 0
max_attempts = 5
while True:
try:
return func(*args, **kwargs)
except Exception as error: # pylint: disable=broad-except
status_code = getattr(error, "status_code", None)
response = getattr(error, "response", None)
if status_code is None and response is not None:
status_code = getattr(response, "status_code", None)
if status_code == 429 and attempts < max_attempts:
delay = _get_retry_delay(response, attempts)
attempts += 1
print(f"Rate limit hit for {func.__name__}. Waiting {delay} seconds before retry {attempts}/{max_attempts}...")
time.sleep(delay)
continue
raise
def fetch_all_projects(api): def fetch_all_projects(api):
ret = [] projects_by_id = {}
projects_iter = api.get_projects()
for projects in projects_iter:
for project in projects:
name = getattr(project, 'name', None)
id = getattr(project, 'id', None)
print(f"Found project {name} with ID {id}")
ret.append(project)
return ret
def fetch_all_completed_tasks(api, project_id):
# Fetch all completed tasks for a project using get_completed_tasks_by_completion_date
# The API only allows up to 3 months per call, so we fetch just once for the last 3 months
all_completed = []
since = (datetime.now() - timedelta(days=90)).replace(hour=0, minute=0, second=0, microsecond=0)
until = datetime.now()
try: try:
completed_iter = api.get_completed_tasks_by_completion_date(since=since, until=until) projects_iter = execute_with_rate_limit(api.get_projects)
for completed_list in completed_iter: for batch in projects_iter:
for task in completed_list: for project in batch:
if hasattr(task, 'project_id') and str(task.project_id) == str(project_id): projects_by_id[str(getattr(project, "id", ""))] = project
all_completed.append(task) except Exception as error: # pylint: disable=broad-except
except Exception as e: print(f"Error fetching projects: {error}")
print(f"Error fetching completed tasks for {since} to {until}: {e}") return list(projects_by_id.values())
print(f"Found {len(all_completed)} completed tasks for project {project_id}")
return all_completed
def fetch_all_tasks(api, project_id, completed=False):
if completed: def fetch_active_tasks_by_project(api):
return fetch_all_completed_tasks(api, project_id) tasks_by_project = defaultdict(list)
else:
tasks = []
try: try:
tasks_iter = api.get_tasks(project_id=project_id) tasks_iter = execute_with_rate_limit(api.get_tasks)
for batch in tasks_iter: for batch in tasks_iter:
for task in batch: for task in batch:
tasks.append(task) tasks_by_project[str(getattr(task, "project_id", ""))].append(task)
except Exception as e: except Exception as error: # pylint: disable=broad-except
print(f"Error fetching active tasks for project {project_id}: {e}") print(f"Error fetching active tasks: {error}")
print(f"Found {len(tasks)} completed tasks for project {project_id}") print(f"Fetched active tasks for {len(tasks_by_project)} projects")
return tasks return tasks_by_project
def fetch_comments(api, task_id): def fetch_completed_tasks_by_project(api, since, until):
comments = [] tasks_by_project = defaultdict(list)
try: try:
comments_iter = api.get_comments(task_id=task_id) completed_iter = execute_with_rate_limit(
api.get_completed_tasks_by_completion_date,
since=since,
until=until,
)
for batch in completed_iter:
for task in batch:
tasks_by_project[str(getattr(task, "project_id", ""))].append(task)
except Exception as error: # pylint: disable=broad-except
print(f"Error fetching completed tasks between {since} and {until}: {error}")
print(f"Fetched completed tasks for {len(tasks_by_project)} projects")
return tasks_by_project
def fetch_comments_by_task(api, project_ids):
comments_by_task = defaultdict(list)
for project_id in project_ids:
try:
comments_iter = execute_with_rate_limit(api.get_comments, project_id=project_id)
for batch in comments_iter: for batch in comments_iter:
for comment in batch: for comment in batch:
comments.append(comment) task_id = str(getattr(comment, "task_id", ""))
except Exception: if task_id:
return [] comments_by_task[task_id].append(comment)
return comments except Exception as error: # pylint: disable=broad-except
print(f"Error fetching comments for project {project_id}: {error}")
print(f"Fetched comments for {len(comments_by_task)} tasks")
return comments_by_task
def process_task(api, task, completed=False): def process_task(task, comments_lookup):
task_dict = task.__dict__.copy() task_dict = task.__dict__.copy()
task_id = getattr(task, "id", None) or getattr(task, "task_id", None)
if task_id is not None:
task_dict.setdefault("id", task_id)
# Attachments (if any) # Attachments (if any)
attachments = [] attachments = []
if hasattr(task, 'attachments') and task.attachments: if hasattr(task, 'attachments') and task.attachments:
@ -123,9 +169,9 @@ def process_task(api, task, completed=False):
if attachments: if attachments:
task_dict['attachments'] = attachments task_dict['attachments'] = attachments
# Comments # Comments
comments = fetch_comments(api, task.id) comment_key = str(task_id) if task_id is not None else None
if comments: if comment_key and comment_key in comments_lookup:
task_dict['comments'] = [c.__dict__ for c in comments] task_dict['comments'] = [c.__dict__ for c in comments_lookup[comment_key]]
return task_dict return task_dict
@ -136,16 +182,22 @@ def main():
ensure_attachments_dir() ensure_attachments_dir()
api = TodoistAPI(get_api_key()) api = TodoistAPI(get_api_key())
projects = fetch_all_projects(api) projects = fetch_all_projects(api)
since = (datetime.now() - timedelta(days=90)).replace(hour=0, minute=0, second=0, microsecond=0)
until = datetime.now()
active_tasks_by_project = fetch_active_tasks_by_project(api)
completed_tasks_by_project = fetch_completed_tasks_by_project(api, since=since, until=until)
comment_project_ids = sorted(
pid for pid in (set(active_tasks_by_project.keys()) | set(completed_tasks_by_project.keys())) if pid
)
comments_by_task = fetch_comments_by_task(api, comment_project_ids)
data = [] data = []
for project in projects: for project in projects:
project_dict = project.__dict__.copy() project_dict = project.__dict__.copy()
project_id = project.id project_id = str(getattr(project, "id", ""))
# Active tasks active_tasks = active_tasks_by_project.get(project_id, [])
active_tasks = fetch_all_tasks(api, project_id, completed=False) completed_tasks = completed_tasks_by_project.get(project_id, [])
# Completed tasks project_dict['tasks'] = [process_task(t, comments_by_task) for t in active_tasks]
completed_tasks = fetch_all_tasks(api, project_id, completed=True) project_dict['completed_tasks'] = [process_task(t, comments_by_task) for t in completed_tasks]
project_dict['tasks'] = [process_task(api, t, completed=False) for t in active_tasks]
project_dict['completed_tasks'] = [process_task(api, t, completed=True) for t in completed_tasks]
data.append(project_dict) data.append(project_dict)
# Write JSON # Write JSON
today = datetime.now().strftime("%Y-%m-%d") today = datetime.now().strftime("%Y-%m-%d")