ealmeida/scripts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init: scripts diversos (crawlers, conversores, scrapers)

Browse Source
This commit is contained in:
Emanuel Almeida
2026-03-05 20:38:36 +00:00
commit 6ac6f4be2a
925 changed files with 850330 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

221
scraper/ctf_config.json Executable file
View File

@@ -0,0 +1,221 @@
{
"client": "CTF_Carstuff",
"output_base_dir": "/media/ealmeida/Dados/GDrive/Cloud/Clientes_360/CTF_Carstuff/KB/Scrapper/sites",
"output_dirs": {
"raw": "output_md",
"cleaned": "output_cleaned",
"formatted": "formatted",
"logs": "logs"
},
"sites": [
{
"name": "The Hog Ring",
"url": "https://www.thehogring.com/",
"type": "wordpress",
"max_depth": 2,
"priority": "high",
"notes": "Blog principal sobre estofamento automotivo"
},
{
"name": "Sailrite",
"url": "https://sailrite.com",
"type": "ecommerce",
"max_depth": 1,
"priority": "medium",
"notes": "E-commerce - focar em blog/recursos"
},
{
"name": "Albright Supply",
"url": "https://albrightssupply.com",
"type": "ecommerce",
"max_depth": 1,
"priority": "medium",
"notes": "E-commerce - focar em recursos educativos"
},
{
"name": "Relicate",
"url": "https://relicate.com",
"type": "website",
"max_depth": 2,
"priority": "medium"
},
{
"name": "TheSamba Main",
"url": "https://thesamba.com",
"type": "website",
"max_depth": 2,
"priority": "high",
"notes": "Recurso VW clássicos"
},
{
"name": "TheSamba VW Forum",
"url": "https://thesamba.com/vw/forum/",
"type": "forum",
"max_depth": 3,
"priority": "high",
"notes": "Fórum VW - comunidade ativa - DEPTH 3 para captura completa"
},
{
"name": "Pelican Parts Forum",
"url": "https://forums.pelicanparts.com",
"type": "forum",
"max_depth": 3,
"priority": "medium",
"notes": "Fórum Porsche - cuidado com rate limits - DEPTH 3 para captura completa"
},
{
"name": "Portal dos Clássicos",
"url": "https://forum.portaldosclassicos.com",
"type": "forum",
"max_depth": 3,
"priority": "high",
"language": "pt",
"notes": "Fórum PT - prioridade alta (mercado local) - DEPTH 3 para captura completa"
},
{
"name": "MG Experience Forum",
"url": "https://mgexp.com/forum",
"type": "forum",
"max_depth": 3,
"priority": "medium",
"notes": "Fórum MG clássicos - DEPTH 3 para captura completa"
},
{
"name": "Triumph Experience Forum",
"url": "https://triumphexp.com/forum/",
"type": "forum",
"max_depth": 3,
"priority": "medium",
"notes": "Fórum Triumph - DEPTH 3 para captura completa"
},
{
"name": "Alfa BB Forums",
"url": "https://alfabb.com/forums",
"type": "forum",
"max_depth": 3,
"priority": "medium",
"notes": "Fórum Alfa Romeo - DEPTH 3 para captura completa"
},
{
"name": "Cruisers Forum",
"url": "https://cruisersforum.com",
"type": "forum",
"max_depth": 3,
"priority": "low",
"notes": "Fórum marítimo - estofamento barcos - DEPTH 3 (MUITO GRANDE, pode demorar 12h+)"
},
{
"name": "Trawler Forum",
"url": "https://trawlerforum.com",
"type": "forum",
"max_depth": 3,
"priority": "low",
"notes": "Fórum marítimo - estofamento barcos - DEPTH 3 (MUITO GRANDE, pode demorar 8h+)"
},
{
"name": "Vans Air Force",
"url": "https://vansairforce.net",
"type": "forum",
"max_depth": 3,
"priority": "low",
"notes": "Fórum aviação - estofamento aeronaves - DEPTH 3 para captura completa"
},
{
"name": "Keyston Bros",
"url": "https://keystonbros.com",
"type": "ecommerce",
"max_depth": 1,
"priority": "low",
"notes": "E-commerce - possível anti-bot (executar com cautela)"
},
{
"name": "Ultrafabrics",
"url": "https://ultrafabricsinc.com",
"type": "ecommerce",
"max_depth": 1,
"priority": "low",
"notes": "E-commerce - possível anti-bot (executar com cautela)"
},
{
"name": "Camira Fabrics",
"url": "https://www.camirafabrics.com",
"type": "ecommerce",
"max_depth": 1,
"priority": "medium",
"notes": "Fornecedor tecidos - catálogo técnico"
},
{
"name": "Sunbrella",
"url": "https://www.sunbrella.com",
"type": "ecommerce",
"max_depth": 1,
"priority": "medium",
"notes": "Fornecedor tecidos - recursos e guias"
}
],
"reddit_subreddits": [
"Autoupholstery",
"upholstery"
],
"german_sites": [
{
"name": "Autosattler.de Community",
"url": "https://autosattler.de/community",
"type": "forum",
"language": "de",
"max_depth": 3,
"priority": "low",
"notes": "Alemão - comunidade estofadores - DEPTH 3 para captura completa (requer validação URL)"
},
{
"name": "Lederzentrum Forum",
"url": "https://lederzentrum.de/forum",
"type": "forum",
"language": "de",
"max_depth": 3,
"priority": "low",
"notes": "Alemão - fórum técnico couro - DEPTH 3 para captura completa"
}
],
"spanish_sites": [
{
"name": "Foro Piel de Toro",
"url": "https://foro.pieldetoro.net",
"type": "forum",
"language": "es",
"max_depth": 3,
"priority": "low",
"notes": "Espanhol - automóveis clássicos - DEPTH 3 para captura completa"
}
],
"scraper_settings": {
"request_timeout": 90,
"max_retries": 3,
"politeness_delay": [3, 8],
"excluded_patterns": [
"/tag/",
"/category/",
"/author/",
"/page/",
"/wp-content/",
"/wp-admin/",
"/feed/",
"/rss/",
"/login",
"/register",
"/cart",
"/checkout",
"/product/",
"/shop/",
"/store/"
]
},
"execution_notes": [
"Sites priority 'high': Executar primeiro",
"Sites priority 'low': Executar por último (maior risco anti-bot)",
"Fóruns: Muito conteúdo, considerar executar separadamente",
"E-commerce: Focar apenas em blog/recursos/guias",
"Reddit: Usar API separada (reddit_scraper.py)",
"Sites alemães/espanhóis: Considerar tradução posterior"
]
}