{"openapi":"3.1.0","info":{"title":"DataFuel API","summary":"DataFuel API is a simple API that allows you to turn website into LLM-ready data","description":"DataFuel API is a simple API that allows you to turn website into LLM-ready data","version":"0.0.2"},"servers":[{"url":"https://api.datafuel.dev"}],"paths":{"/ping":{"get":{"summary":"Hello","operationId":"hello_ping_get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/":{"get":{"summary":"Root","operationId":"root__get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}},"/scrape":{"post":{"summary":"Scrape","operationId":"scrape_scrape_post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ScrapeRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ScrapeResponse"}}}},"401":{"description":"Unauthorized"},"400":{"description":"Bad Request"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}},"security":[{"HTTPBearer":[]}]}},"/crawl":{"post":{"summary":"Crawl","operationId":"crawl_crawl_post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/CrawlRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ScrapeResponse"}}}},"401":{"description":"Unauthorized"},"400":{"description":"Bad Request"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}},"security":[{"HTTPBearer":[]}]}},"/scrape_with_login":{"post":{"summary":"Scrape With Login","operationId":"scrape_with_login_scrape_with_login_post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/ScrapeWithLoginRequest"}}},"required":true},"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/ScrapeResponse"}}}},"401":{"description":"Unauthorized"},"400":{"description":"Bad Request"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}},"security":[{"HTTPBearer":[]}]}},"/list_scrapes":{"get":{"summary":"Get Scrapes Metadata","operationId":"get_scrapes_metadata_list_scrapes_get","security":[{"HTTPBearer":[]}],"parameters":[{"name":"job_id","in":"query","required":false,"schema":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"Optional job ID to filter scrapes","title":"Job Id"},"description":"Optional job ID to filter scrapes"},{"name":"markdown","in":"query","required":false,"schema":{"type":"boolean","description":"If true, return markdown data on top of the signed_url","default":false,"title":"Markdown"},"description":"If true, return markdown data on top of the signed_url"},{"name":"ai_response","in":"query","required":false,"schema":{"type":"boolean","description":"If true, return AI response data on top of the signed_url","default":false,"title":"Ai Response"},"description":"If true, return AI response data on top of the signed_url"},{"name":"url","in":"query","required":false,"schema":{"anyOf":[{"type":"string"},{"type":"null"}],"description":"If provided, filter scrapes by the given URL","title":"Url"},"description":"If provided, filter scrapes by the given URL"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"type":"array","items":{"$ref":"#/components/schemas/ScrapeMetadata"},"title":"Response Get Scrapes Metadata List Scrapes Get"}}}},"401":{"description":"Unauthorized"},"404":{"description":"Not Found"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/me":{"get":{"summary":"Get User Info","operationId":"get_user_info_me_get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"$ref":"#/components/schemas/UserInfo"}}}},"401":{"description":"Unauthorized"}},"security":[{"HTTPBearer":[]}]}},"/list_jobs":{"get":{"summary":"Get Job Ids","operationId":"get_job_ids_list_jobs_get","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{"items":{"$ref":"#/components/schemas/JobId"},"type":"array","title":"Response Get Job Ids List Jobs Get"}}}},"401":{"description":"Unauthorized"},"404":{"description":"No jobs found"}},"security":[{"HTTPBearer":[]}]}},"/webhook/subscribe":{"post":{"summary":"Subscribe Webhook","operationId":"subscribe_webhook_webhook_subscribe_post","requestBody":{"content":{"application/json":{"schema":{"$ref":"#/components/schemas/WebhookSubscription"}}},"required":true},"responses":{"201":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"401":{"description":"Unauthorized"},"400":{"description":"Invalid webhook URL"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}},"security":[{"HTTPBearer":[]}]}},"/webhook/unsubscribe":{"delete":{"summary":"Unsubscribe Webhook","operationId":"unsubscribe_webhook_webhook_unsubscribe_delete","security":[{"HTTPBearer":[]}],"parameters":[{"name":"webhook_url","in":"query","required":true,"schema":{"type":"string","description":"URL to unsubscribe from webhook notifications","title":"Webhook Url"},"description":"URL to unsubscribe from webhook notifications"}],"responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}},"401":{"description":"Unauthorized"},"404":{"description":"Webhook URL not found"},"422":{"description":"Validation Error","content":{"application/json":{"schema":{"$ref":"#/components/schemas/HTTPValidationError"}}}}}}},"/webhook":{"post":{"summary":"Webhook Handler","operationId":"webhook_handler_webhook_post","responses":{"200":{"description":"Successful Response","content":{"application/json":{"schema":{}}}}}}}},"components":{"schemas":{"CrawlRequest":{"properties":{"url":{"type":"string","title":"Url"},"ai_prompt":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Ai Prompt"},"json_schema":{"anyOf":[{"type":"object"},{"type":"null"}],"title":"Json Schema","description":"Optional schema definition for structured data extraction. \nFormat should follow OpenAI's function calling schema format (https://platform.openai.com/docs/guides/structured-outputs).\n\nExample types:\n- string: \"type\": \"string\"\n- integer: \"type\": \"integer\"\n- number: \"type\": \"number\"\n- boolean: \"type\": \"boolean\"\n- array: \"type\": \"array\", \"items\": {\"type\": \"string\"}\n- object: \"type\": \"object\", \"properties\": {...}","example":{"description":"Schema for capturing product information","name":"Product Schema","schema":{"properties":{"product_url":{"description":"The URL of the specific product","type":"string"},"product_name":{"description":"The name of the specific product","type":"string"},"price":{"description":"The price of the product","type":"number"},"product_images":{"description":"List of product image URLs","items":{"properties":{"url":{"description":"URL of the product image","type":"string"}},"required":["url"],"type":"object"},"type":"array"}},"required":["product_url","product_name","price","product_images"],"type":"object"}}},"javascript_scenario":{"anyOf":[{"items":{"type":"object"},"type":"array"},{"type":"null"}],"title":"Javascript Scenario"},"depth":{"type":"integer","title":"Depth","description":"The depth of the crawl 1 depth mean only the first level of links will be scraped like https://example.com/page1 and https://example.com/page2","default":1,"example":1},"limit":{"type":"integer","title":"Limit","description":"The maximum number of pages to scrape","default":1,"example":1},"exclusion_pattern":{"type":"string","title":"Exclusion Pattern","description":"Regex pattern to exclude specific URLs (e.g., 'https://.*datafuel\\.dev/blog/.*' to exclude blog pages)","default":"","example":"https://.*datafuel\\.dev/blog/.*"},"excluded_links":{"type":"string","title":"Excluded Links","description":"Comma-separated list of URLs to exclude from crawling","default":"","example":"https://www.datafuel.dev/pricing,https://www.datafuel.dev/blog"}},"type":"object","required":["url"],"title":"CrawlRequest"},"HTTPValidationError":{"properties":{"detail":{"items":{"$ref":"#/components/schemas/ValidationError"},"type":"array","title":"Detail"}},"type":"object","title":"HTTPValidationError"},"JobId":{"properties":{"id":{"type":"string","title":"Id","description":"Unique job ID associated with the API key","example":"f47ac10b-58cc-4372-a567-0e02b2c3d479"}},"type":"object","required":["id"],"title":"JobId"},"JobStatus":{"type":"string","enum":["pending","finished"],"title":"JobStatus"},"ScrapeMetadata":{"properties":{"job_id":{"type":"string","title":"Job Id","description":"The unique identifier for the scraping job that can contain multiple scrapes","example":"f47ac10b-58cc-4372-a567-0e02b2c3d479"},"scrape_id":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Scrape Id","description":"The unique identifier for the each URL scrape","example":"550e8400-e29b-41d4-a716-446655440000"},"scrape_status":{"$ref":"#/components/schemas/ScrapeStatus"},"scrape_url":{"type":"string","title":"Scrape Url","description":"The URL that was scraped","example":"https://example.com"},"scrape_timestamp":{"type":"string","title":"Scrape Timestamp","description":"The timestamp of when the scrape was created","example":"2024-03-15T10:30:00Z"},"signed_url":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Signed Url","description":"Pre-signed URL containing the scraping results in JSON format with structure:\n{\n    \"ai_response\": \"AI-generated analysis or structured data\",\n    \"markdown\": \"Content formatted in markdown\",\n    \"html\": \"Raw HTML content from the scraped page\"\n}","example":"https://docyjgyvimrauivbukcp.supabase.co/storage/v1/object/sign/scrapes-data/..."},"ai_response":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Ai Response","description":"AI-generated analysis or structured data"},"markdown":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Markdown","description":"Content formatted in markdown"},"job_status":{"$ref":"#/components/schemas/JobStatus"}},"type":"object","required":["job_id","scrape_status","scrape_url","scrape_timestamp","job_status"],"title":"ScrapeMetadata","example":{"job_id":"f47ac10b-58cc-4372-a567-0e02b2c3d479","job_status":"finished","scrape_id":"550e8400-e29b-41d4-a716-446655440000","scrape_status":"success","scrape_timestamp":"2024-03-15T10:30:00Z","scrape_url":"https://example.com","signed_url":"https://docyjgyvimrauivbukcp.supabase.co/storage/v1/object/sign/scrapes-data/..."},"signed_url_response_format":{"description":"The signed_url returns a JSON with the following structure:","schema":{"ai_response":"AI-generated analysis or structured data","html":"Raw HTML content from the scraped page","markdown":"Content formatted in markdown"}}},"ScrapeRequest":{"properties":{"url":{"type":"string","title":"Url"},"ai_prompt":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Ai Prompt"},"json_schema":{"anyOf":[{"type":"object"},{"type":"null"}],"title":"Json Schema","description":"Optional schema definition for structured data extraction. \nFormat should follow OpenAI's function calling schema format (https://platform.openai.com/docs/guides/structured-outputs).\n\nExample types:\n- string: \"type\": \"string\"\n- integer: \"type\": \"integer\"\n- number: \"type\": \"number\"\n- boolean: \"type\": \"boolean\"\n- array: \"type\": \"array\", \"items\": {\"type\": \"string\"}\n- object: \"type\": \"object\", \"properties\": {...}","example":{"description":"Schema for capturing product information","name":"Product Schema","schema":{"properties":{"product_url":{"description":"The URL of the specific product","type":"string"},"product_name":{"description":"The name of the specific product","type":"string"},"price":{"description":"The price of the product","type":"number"},"product_images":{"description":"List of product image URLs","items":{"properties":{"url":{"description":"URL of the product image","type":"string"}},"required":["url"],"type":"object"},"type":"array"}},"required":["product_url","product_name","price","product_images"],"type":"object"}}},"javascript_scenario":{"anyOf":[{"items":{"type":"object"},"type":"array"},{"type":"null"}],"title":"Javascript Scenario"}},"type":"object","required":["url"],"title":"ScrapeRequest"},"ScrapeResponse":{"properties":{"job_id":{"type":"string","title":"Job Id","description":"The identifier for the scraping job","example":"f47ac10b-58cc-4372-a567-0e02b2c3d479"}},"type":"object","required":["job_id"],"title":"ScrapeResponse"},"ScrapeStatus":{"type":"string","enum":["pending","failed","success"],"title":"ScrapeStatus"},"ScrapeWithLoginRequest":{"properties":{"url":{"type":"string","title":"Url"},"ai_prompt":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Ai Prompt"},"json_schema":{"anyOf":[{"type":"object"},{"type":"null"}],"title":"Json Schema","description":"Optional schema definition for structured data extraction. \nFormat should follow OpenAI's function calling schema format (https://platform.openai.com/docs/guides/structured-outputs).\n\nExample types:\n- string: \"type\": \"string\"\n- integer: \"type\": \"integer\"\n- number: \"type\": \"number\"\n- boolean: \"type\": \"boolean\"\n- array: \"type\": \"array\", \"items\": {\"type\": \"string\"}\n- object: \"type\": \"object\", \"properties\": {...}","example":{"description":"Schema for capturing product information","name":"Product Schema","schema":{"properties":{"product_url":{"description":"The URL of the specific product","type":"string"},"product_name":{"description":"The name of the specific product","type":"string"},"price":{"description":"The price of the product","type":"number"},"product_images":{"description":"List of product image URLs","items":{"properties":{"url":{"description":"URL of the product image","type":"string"}},"required":["url"],"type":"object"},"type":"array"}},"required":["product_url","product_name","price","product_images"],"type":"object"}}},"javascript_scenario":{"anyOf":[{"items":{"type":"object"},"type":"array"},{"type":"null"}],"title":"Javascript Scenario"},"username":{"type":"string","title":"Username"},"password":{"type":"string","title":"Password"},"login_url":{"anyOf":[{"type":"string"},{"type":"null"}],"title":"Login Url"}},"type":"object","required":["url","username","password"],"title":"ScrapeWithLoginRequest"},"UserInfo":{"properties":{"user_id":{"type":"string","title":"User Id"},"is_valid":{"type":"boolean","title":"Is Valid","default":true}},"type":"object","required":["user_id"],"title":"UserInfo"},"ValidationError":{"properties":{"loc":{"items":{"anyOf":[{"type":"string"},{"type":"integer"}]},"type":"array","title":"Location"},"msg":{"type":"string","title":"Message"},"type":{"type":"string","title":"Error Type"}},"type":"object","required":["loc","msg","type"],"title":"ValidationError"},"WebhookSubscription":{"properties":{"webhook_url":{"type":"string","title":"Webhook Url","description":"URL to receive webhook notifications","example":"https://api.example.com/webhook"}},"type":"object","required":["webhook_url"],"title":"WebhookSubscription"}},"securitySchemes":{"HTTPBearer":{"type":"http","scheme":"bearer"}}}}