Daily News Digest with LLM Deduplication

import json
import time
from datetime import date
from pathlib import Path
from perplexity import Perplexity

client = Perplexity()

TOPICS = [
    "data center construction and expansion news",
    "renewable energy project financing news",
    "commercial real estate transaction news",
]

STATE_FILE = Path("stories.json")
known = json.loads(STATE_FILE.read_text()) if STATE_FILE.exists() else {}
known_list = "\n".join(
    f"- {story_id}: {story['headline']} (last seen {story['last_seen']})"
    for story_id, story in known.items()
) or "(none)"
topic_list = "\n".join(f"- {topic}" for topic in TOPICS)

INSTRUCTIONS = """You build a daily news digest in the sandbox.

Workflow:
1. In one sandbox execution, write code that searches each topic the user gives you with the Perplexity SDK, one search per topic, max_results 20, restricted to the last day. Make exactly one search per topic and no other searches. Do not fetch article pages. Save the collected results (title, url, date, snippet) to results.json and print a compact numbered summary of them.
2. Read the printed summary and group the results into stories yourself. Grouping is judgment, not code: do not group with string matching or embeddings.
3. In one final sandbox execution, write code that loads results.json and builds two files from it and your grouping, named exactly digest.md and stories.json (refer to results by their numbers so code copies titles and urls exactly; never retype a url):
   - digest.md: one section per story that is new or an update. End an update story's headline with a single (update) marker. Include the headline, a one or two sentence summary, a markdown link to the primary article, and a line listing the other domains that covered it.
   - stories.json: the known stories the user gives you plus every story from today, as {"story_id": {"headline": ..., "last_seen": "YYYY-MM-DD"}}. Set last_seen to today only for stories that appeared in today's results; keep the previous last_seen for the rest.
   Then share both files with the share_file tool.

Grouping rules:
- Two articles are the same story if they report the same underlying event. Republished, reworded, or partial coverage of one event is one story.
- Similar language is not enough. Recurring events are different stories: earnings for different quarters, reports for different months, deals involving different companies.
- Ignore results that are not coverage of a single event, such as homepages, category pages, and link roundups.
- The primary article is the most complete or most original source in the group. Prefer a dedicated article page from the original outlet over aggregators and republished copies.
- story_id is a short lowercase name for the underlying event, with words separated by hyphens, like aws-chile-region. If the event matches a known story the user lists, reuse that story_id exactly.
- A story is "new" if the event is not in the known stories, an "update" if it matches a known story and adds material new information, and a "repeat" if it matches a known story and adds nothing. Leave repeats out of digest.md but keep them in stories.json."""

response = client.responses.create(
    model="anthropic/claude-haiku-4-5",
    max_output_tokens=32000,
    max_steps=50,
    background=True,
    tools=[{"type": "sandbox"}],
    instructions=INSTRUCTIONS,
    input=f"""Build the digest for {date.today():%Y-%m-%d}. Topics:
{topic_list}

Known stories from previous digests:
{known_list}""",
)

while response.status in ("queued", "in_progress"):
    time.sleep(5)
    response = client.responses.retrieve(response.id)

print(f"Final status: {response.status}")

files = client.responses.files.list(response.id)
for file in files.data:
    content = client.responses.files.content(
        file_id=file.id,
        response_id=response.id,
    )
    content.write_to_file(file.filename)
    print(f"Downloaded {file.filename} ({file.bytes} bytes)")

import Perplexity from '@perplexity-ai/perplexity_ai';
import { readFile, writeFile } from 'node:fs/promises';

const client = new Perplexity();

const TOPICS = [
  'data center construction and expansion news',
  'renewable energy project financing news',
  'commercial real estate transaction news',
];

let known: Record<string, { headline: string; last_seen: string }> = {};
try {
  known = JSON.parse(await readFile('stories.json', 'utf8'));
} catch {}
const knownList =
  Object.entries(known)
    .map(([storyId, story]) => `- ${storyId}: ${story.headline} (last seen ${story.last_seen})`)
    .join('\n') || '(none)';
const topicList = TOPICS.map((topic) => `- ${topic}`).join('\n');

const INSTRUCTIONS = `You build a daily news digest in the sandbox.

Workflow:
1. In one sandbox execution, write code that searches each topic the user gives you with the Perplexity SDK, one search per topic, max_results 20, restricted to the last day. Make exactly one search per topic and no other searches. Do not fetch article pages. Save the collected results (title, url, date, snippet) to results.json and print a compact numbered summary of them.
2. Read the printed summary and group the results into stories yourself. Grouping is judgment, not code: do not group with string matching or embeddings.
3. In one final sandbox execution, write code that loads results.json and builds two files from it and your grouping, named exactly digest.md and stories.json (refer to results by their numbers so code copies titles and urls exactly; never retype a url):
   - digest.md: one section per story that is new or an update. End an update story's headline with a single (update) marker. Include the headline, a one or two sentence summary, a markdown link to the primary article, and a line listing the other domains that covered it.
   - stories.json: the known stories the user gives you plus every story from today, as {"story_id": {"headline": ..., "last_seen": "YYYY-MM-DD"}}. Set last_seen to today only for stories that appeared in today's results; keep the previous last_seen for the rest.
   Then share both files with the share_file tool.

Grouping rules:
- Two articles are the same story if they report the same underlying event. Republished, reworded, or partial coverage of one event is one story.
- Similar language is not enough. Recurring events are different stories: earnings for different quarters, reports for different months, deals involving different companies.
- Ignore results that are not coverage of a single event, such as homepages, category pages, and link roundups.
- The primary article is the most complete or most original source in the group. Prefer a dedicated article page from the original outlet over aggregators and republished copies.
- story_id is a short lowercase name for the underlying event, with words separated by hyphens, like aws-chile-region. If the event matches a known story the user lists, reuse that story_id exactly.
- A story is "new" if the event is not in the known stories, an "update" if it matches a known story and adds material new information, and a "repeat" if it matches a known story and adds nothing. Leave repeats out of digest.md but keep them in stories.json.`;

const today = new Date().toISOString().slice(0, 10);

let response = await client.responses.create({
  model: 'anthropic/claude-haiku-4-5',
  max_output_tokens: 32000,
  max_steps: 50,
  background: true,
  tools: [{ type: 'sandbox' }],
  instructions: INSTRUCTIONS,
  input: `Build the digest for ${today}. Topics:
${topicList}

Known stories from previous digests:
${knownList}`,
});

while (response.status === 'queued' || response.status === 'in_progress') {
  await new Promise((resolve) => setTimeout(resolve, 5000));
  response = await client.responses.retrieve(response.id);
}

console.log(`Final status: ${response.status}`);

const files = await client.responses.files.list(response.id);
for (const file of files.data) {
  const content = await client.responses.files.content(file.id, {
    response_id: response.id,
  });
  await writeFile(file.filename, Buffer.from(await content.arrayBuffer()));
  console.log(`Downloaded ${file.filename} (${file.bytes} bytes)`);
}