init: scripts diversos (crawlers, conversores, scrapers)

2026-03-05 20:38:36 +00:00
commit 6ac6f4be2a
925 changed files with 850330 additions and 0 deletions
--- a/scraper/sites_config.json
+++ b/scraper/sites_config.json
@@ -0,0 +1,162 @@
+{
+  "sites": [
+    {
+      "name": "The Hog Ring",
+      "url": "https://www.thehogring.com/",
+      "type": "wordpress",
+      "max_depth": 2,
+      "notes": "Blog principal sobre estofamento automotivo"
+    },
+    {
+      "name": "Sailrite",
+      "url": "https://sailrite.com",
+      "type": "ecommerce",
+      "max_depth": 1,
+      "notes": "E-commerce - focar em blog/recursos"
+    },
+    {
+      "name": "Albright Supply",
+      "url": "https://albrightssupply.com",
+      "type": "ecommerce",
+      "max_depth": 1,
+      "notes": "E-commerce - focar em recursos educativos"
+    },
+    {
+      "name": "Relicate",
+      "url": "https://relicate.com",
+      "type": "website",
+      "max_depth": 2
+    },
+    {
+      "name": "TheSamba Main",
+      "url": "https://thesamba.com",
+      "type": "website",
+      "max_depth": 2
+    },
+    {
+      "name": "TheSamba VW Forum",
+      "url": "https://thesamba.com/vw/forum/",
+      "type": "forum",
+      "max_depth": 1,
+      "notes": "Fórum - usar scraper específico"
+    },
+    {
+      "name": "Pelican Parts Forum",
+      "url": "https://forums.pelicanparts.com",
+      "type": "forum",
+      "max_depth": 1,
+      "notes": "Fórum - cuidado com rate limits"
+    },
+    {
+      "name": "Portal dos Clássicos",
+      "url": "https://forum.portaldosclassicos.com",
+      "type": "forum",
+      "max_depth": 1,
+      "notes": "Fórum PT - prioridade média"
+    },
+    {
+      "name": "MG Experience Forum",
+      "url": "https://mgexp.com/forum",
+      "type": "forum",
+      "max_depth": 1
+    },
+    {
+      "name": "Triumph Experience Forum",
+      "url": "https://triumphexp.com/forum/",
+      "type": "forum",
+      "max_depth": 1
+    },
+    {
+      "name": "Alfa BB Forums",
+      "url": "https://alfabb.com/forums",
+      "type": "forum",
+      "max_depth": 1
+    },
+    {
+      "name": "Cruisers Forum",
+      "url": "https://cruisersforum.com",
+      "type": "forum",
+      "max_depth": 1,
+      "notes": "Fórum marítimo"
+    },
+    {
+      "name": "Trawler Forum",
+      "url": "https://trawlerforum.com",
+      "type": "forum",
+      "max_depth": 1,
+      "notes": "Fórum marítimo"
+    },
+    {
+      "name": "Vans Air Force",
+      "url": "https://vansairforce.net",
+      "type": "forum",
+      "max_depth": 1,
+      "notes": "Fórum aviação"
+    },
+    {
+      "name": "Keyston Bros",
+      "url": "https://keystonbros.com",
+      "type": "ecommerce",
+      "max_depth": 1,
+      "notes": "E-commerce - possível anti-bot"
+    },
+    {
+      "name": "Ultrafabrics",
+      "url": "https://ultrafabricsinc.com",
+      "type": "ecommerce",
+      "max_depth": 1,
+      "notes": "E-commerce - possível anti-bot"
+    }
+  ],
+  "reddit_subreddits": [
+    "Autoupholstery",
+    "upholstery"
+  ],
+  "german_sites": [
+    {
+      "name": "Autosattler.de Community",
+      "url": "https://autosattler.de/community",
+      "type": "forum",
+      "language": "de",
+      "notes": "Alemão - pode requerer tradução"
+    },
+    {
+      "name": "Lederzentrum Forum",
+      "url": "https://lederzentrum.de/forum",
+      "type": "forum",
+      "language": "de",
+      "notes": "Alemão - fórum técnico"
+    }
+  ],
+  "spanish_sites": [
+    {
+      "name": "Foro Piel de Toro",
+      "url": "https://foro.pieldetoro.net",
+      "type": "forum",
+      "language": "es",
+      "notes": "Espanhol - automóveis clássicos"
+    }
+  ],
+  "fabric_suppliers": [
+    {
+      "name": "Camira Fabrics",
+      "url": "https://www.camirafabrics.com",
+      "type": "ecommerce",
+      "max_depth": 1,
+      "notes": "Fornecedor tecidos - catálogo"
+    },
+    {
+      "name": "Sunbrella",
+      "url": "https://www.sunbrella.com",
+      "type": "ecommerce",
+      "max_depth": 1,
+      "notes": "Fornecedor tecidos - recursos"
+    }
+  ],
+  "notes": [
+    "Sites e-commerce: Focar em blog/recursos educativos, não produtos",
+    "Fóruns: max_depth=1 para evitar scraping excessivo",
+    "Reddit: Usar API separada (módulo reddit_scraper.py)",
+    "Sites alemães/espanhóis: Considerar tradução se necessário"
+  ]
+}