@@ -0,0 +1,5 @@ | |||
APP_REDIS_HOST=redis-cnds | |||
APP_REDIS_PORT=6380 | |||
CAP_MONSTER_KEY =89310c8388da93bf17133bf399f517de | |||
TWO_CAPTCHA_KEY=0a26ea85b64ea9797cc296337001f6ab |
@@ -0,0 +1,24 @@ | |||
module.exports = { | |||
parser: '@typescript-eslint/parser', | |||
parserOptions: { | |||
project: 'tsconfig.json', | |||
sourceType: 'module', | |||
}, | |||
plugins: ['@typescript-eslint/eslint-plugin'], | |||
extends: [ | |||
'plugin:@typescript-eslint/recommended', | |||
'plugin:prettier/recommended', | |||
], | |||
root: true, | |||
env: { | |||
node: true, | |||
jest: true, | |||
}, | |||
ignorePatterns: ['.eslintrc.js'], | |||
rules: { | |||
'@typescript-eslint/interface-name-prefix': 'off', | |||
'@typescript-eslint/explicit-function-return-type': 'off', | |||
'@typescript-eslint/explicit-module-boundary-types': 'off', | |||
'@typescript-eslint/no-explicit-any': 'off', | |||
}, | |||
}; |
@@ -0,0 +1,35 @@ | |||
# compiled output | |||
/dist | |||
/node_modules | |||
# Logs | |||
logs | |||
*.log | |||
npm-debug.log* | |||
pnpm-debug.log* | |||
yarn-debug.log* | |||
yarn-error.log* | |||
lerna-debug.log* | |||
# OS | |||
.DS_Store | |||
# Tests | |||
/coverage | |||
/.nyc_output | |||
# IDEs and editors | |||
/.idea | |||
.project | |||
.classpath | |||
.c9/ | |||
*.launch | |||
.settings/ | |||
*.sublime-workspace | |||
# IDE - VSCode | |||
.vscode/* | |||
!.vscode/settings.json | |||
!.vscode/tasks.json | |||
!.vscode/launch.json | |||
!.vscode/extensions.json |
@@ -0,0 +1,4 @@ | |||
{ | |||
"singleQuote": true, | |||
"trailingComma": "all" | |||
} |
@@ -0,0 +1,73 @@ | |||
<p align="center"> | |||
<a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo_text.svg" width="320" alt="Nest Logo" /></a> | |||
</p> | |||
[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456 | |||
[circleci-url]: https://circleci.com/gh/nestjs/nest | |||
<p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p> | |||
<p align="center"> | |||
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a> | |||
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a> | |||
<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a> | |||
<a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a> | |||
<a href="https://coveralls.io/github/nestjs/nest?branch=master" target="_blank"><img src="https://coveralls.io/repos/github/nestjs/nest/badge.svg?branch=master#9" alt="Coverage" /></a> | |||
<a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a> | |||
<a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a> | |||
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a> | |||
<a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg"/></a> | |||
<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a> | |||
<a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow"></a> | |||
</p> | |||
<!--[![Backers on Open Collective](https://opencollective.com/nest/backers/badge.svg)](https://opencollective.com/nest#backer) | |||
[![Sponsors on Open Collective](https://opencollective.com/nest/sponsors/badge.svg)](https://opencollective.com/nest#sponsor)--> | |||
## Description | |||
[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository. | |||
## Installation | |||
```bash | |||
$ npm install | |||
``` | |||
## Running the app | |||
```bash | |||
# development | |||
$ npm run start | |||
# watch mode | |||
$ npm run start:dev | |||
# production mode | |||
$ npm run start:prod | |||
``` | |||
## Test | |||
```bash | |||
# unit tests | |||
$ npm run test | |||
# e2e tests | |||
$ npm run test:e2e | |||
# test coverage | |||
$ npm run test:cov | |||
``` | |||
## Support | |||
Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support). | |||
## Stay in touch | |||
- Author - [Kamil Myśliwiec](https://kamilmysliwiec.com) | |||
- Website - [https://nestjs.com](https://nestjs.com/) | |||
- Twitter - [@nestframework](https://twitter.com/nestframework) | |||
## License | |||
Nest is [MIT licensed](LICENSE). |
@@ -0,0 +1,4 @@ | |||
{ | |||
"collection": "@nestjs/schematics", | |||
"sourceRoot": "src" | |||
} |
@@ -0,0 +1,82 @@ | |||
{ | |||
"name": "scraper-cnd-mt", | |||
"version": "0.0.1", | |||
"description": "", | |||
"author": "", | |||
"private": true, | |||
"license": "UNLICENSED", | |||
"scripts": { | |||
"prebuild": "rimraf dist", | |||
"build": "nest build", | |||
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", | |||
"start": "nest start", | |||
"start:dev": "nest start --watch", | |||
"start:debug": "nest start --debug --watch", | |||
"start:prod": "node dist/main", | |||
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", | |||
"test": "jest", | |||
"test:watch": "jest --watch", | |||
"test:cov": "jest --coverage", | |||
"test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", | |||
"test:e2e": "jest --config ./test/jest-e2e.json", | |||
"docker:build": "", | |||
"docker:tag": "", | |||
"docker:push": "", | |||
"docker:publish": "npm run docker:tag && npm run docker:push" | |||
}, | |||
"dependencies": { | |||
"@infosimples/node_two_captcha": "^1.0.2", | |||
"@nestjs/bull": "^0.1.2", | |||
"@nestjs/common": "^7.6.18", | |||
"@nestjs/config": "^0.4.0", | |||
"@nestjs/core": "^7.0.0", | |||
"@nestjs/platform-express": "^7.0.0", | |||
"bull": "^3.15.0", | |||
"data-fns": "^0.1.8", | |||
"date-fns": "^2.12.0", | |||
"pdf-parse": "^1.1.1", | |||
"puppeteer": "^5.0.0", | |||
"reflect-metadata": "^0.1.13", | |||
"rimraf": "^3.0.2", | |||
"rxjs": "^6.5.4" | |||
}, | |||
"devDependencies": { | |||
"@nestjs/cli": "^7.0.0", | |||
"@nestjs/schematics": "^7.0.0", | |||
"@nestjs/testing": "^7.0.0", | |||
"@types/bull": "^3.14.0", | |||
"@types/express": "^4.17.3", | |||
"@types/jest": "25.1.4", | |||
"@types/node": "^13.9.1", | |||
"@types/pdf-parse": "^1.1.1", | |||
"@types/puppeteer": "^3.0.1", | |||
"@types/supertest": "^2.0.8", | |||
"@typescript-eslint/eslint-plugin": "^2.23.0", | |||
"@typescript-eslint/parser": "^2.23.0", | |||
"eslint": "^6.8.0", | |||
"eslint-config-prettier": "^6.10.0", | |||
"eslint-plugin-import": "^2.20.1", | |||
"jest": "^25.1.0", | |||
"prettier": "^1.19.1", | |||
"supertest": "^4.0.2", | |||
"ts-jest": "25.2.1", | |||
"ts-loader": "^6.2.1", | |||
"ts-node": "^8.6.2", | |||
"tsconfig-paths": "^3.9.0", | |||
"typescript": "^3.7.4" | |||
}, | |||
"jest": { | |||
"moduleFileExtensions": [ | |||
"js", | |||
"json", | |||
"ts" | |||
], | |||
"rootDir": "src", | |||
"testRegex": ".spec.ts$", | |||
"transform": { | |||
"^.+\\.(t|j)s$": "ts-jest" | |||
}, | |||
"coverageDirectory": "../coverage", | |||
"testEnvironment": "node" | |||
} | |||
} |
@@ -0,0 +1,12 @@ | |||
import { Module } from '@nestjs/common'; | |||
import { ConfigModule } from '@nestjs/config'; | |||
import { ScraperCndMtModule } from './modules/scraper-cnd-mt/scraper-cnd-mt.module'; | |||
@Module({ | |||
imports: [ | |||
ConfigModule.forRoot({ isGlobal: true }), | |||
ScraperCndMtModule | |||
], | |||
}) | |||
export class AppModule {} |
@@ -0,0 +1,6 @@ | |||
export enum Situacao { | |||
Negativa = 1, | |||
Positiva, | |||
FalhaConsulta, | |||
PositivaEfeitoNegativa, | |||
} |
@@ -0,0 +1,114 @@ | |||
import axios from 'axios' | |||
import { ICaptchaResolver } from '../interfaces/ICaptchaResolver'; | |||
export class CapMonster implements ICaptchaResolver { | |||
private readonly CAPTCHA_KEY: string = process.env.CAP_MONSTER_KEY | |||
private readonly CAPTCHA_BALANCE: string = | |||
'https://api.capmonster.cloud/getBalance' | |||
private readonly CAPTCHA_CREATE: string = | |||
'https://api.capmonster.cloud/createTask' | |||
private readonly CAPTCHA_RESULT: string = | |||
'https://api.capmonster.cloud/getTaskResult' | |||
public async temSaldoDisponivel (): Promise<boolean> { | |||
try { | |||
const balanceData = { | |||
clientKey: this.CAPTCHA_KEY | |||
} | |||
const { data } = await axios.post(this.CAPTCHA_BALANCE, balanceData) | |||
if (data.errorId > 0) { | |||
console.log(`[consultar-saldo-captcha] - ${data.errorCode}`) | |||
return false | |||
} | |||
return data.balance > 0 | |||
} catch (error) { | |||
console.log(`[consultar-saldo-captcha] - ${error.message}`) | |||
return false | |||
} | |||
} | |||
public async resolver (imageData: string): Promise<string> { | |||
const taskId = await this.criar(imageData) | |||
if (taskId === 0) return null | |||
return await this.resultado(taskId) | |||
} | |||
private async criar (imageData: string): Promise<number> { | |||
try { | |||
const createTask = { | |||
clientKey: this.CAPTCHA_KEY, | |||
task: { | |||
type: 'ImageToTextTask', | |||
body: imageData | |||
} | |||
} | |||
const { data } = await axios.post(this.CAPTCHA_CREATE, createTask) | |||
if (data.errorId > 0) { | |||
console.log(`[criar-captcha] - ${data.errorCode}`) | |||
return 0 | |||
} | |||
return data.taskId | |||
} catch (error) { | |||
console.log(`[criar-captcha] - ${error.message}`) | |||
return 0 | |||
} | |||
} | |||
private async resultado (taskId: number): Promise<string> { | |||
let tentativas = 0 | |||
let error: boolean = false | |||
let captchaData = null | |||
const getResultTask = { | |||
clientKey: this.CAPTCHA_KEY, | |||
taskId: taskId | |||
} | |||
do { | |||
try { | |||
const { | |||
data: { errorId, errorCode, solution, status } | |||
} = await axios.post(this.CAPTCHA_RESULT, getResultTask) | |||
if (errorId > 0) { | |||
console.log( | |||
`[resultado-captcha] - Task: ${taskId} Erro: ${errorCode} Tentativas: ${tentativas}` | |||
) | |||
error = true | |||
break | |||
} | |||
if (status === 'ready') { | |||
captchaData = solution.text | |||
break | |||
} | |||
tentativas += 1 | |||
await new Promise(r => setTimeout(r, 10 * 1000)) | |||
} catch (error) { | |||
console.log( | |||
`[resultado-captcha] - Task: ${taskId} Erro: ${error.message} Tentativas: ${tentativas}` | |||
) | |||
error = true | |||
break | |||
} | |||
} while (tentativas < 20) | |||
if (error) return null | |||
return captchaData | |||
} | |||
} | |||
export default new CapMonster() |
@@ -0,0 +1,114 @@ | |||
import axios from 'axios' | |||
export class TwoCaptcha { | |||
private readonly CAPTCHA_KEY: string = 'TWO_CAPTCHA_KEY' | |||
private readonly CAPTCHA_CREATE: string = 'http://2captcha.com/in.php' | |||
private readonly CAPTCHA_RESULT: string = 'http://2captcha.com/res.php' | |||
private readonly CAPTCHA_BALANCE: string = 'https://2captcha.com/res.php' | |||
async resolveCaptcha (imageData: string) { | |||
const captchaId = await this._sendCaptcha(imageData) | |||
const captchaResolved = await this._getResponseCaptcha(captchaId) | |||
return captchaResolved | |||
} | |||
public async temSaldoDisponivel (): Promise<boolean> { | |||
try { | |||
const balanceData = { | |||
key: this.CAPTCHA_KEY, | |||
action: 'getbalance', | |||
json: 1 | |||
} | |||
const { data: { request } } = await axios.get(this.CAPTCHA_BALANCE, { params: balanceData }); | |||
if ((request || '').includes('ERROR_')) { | |||
console.log(`[consultar-saldo-captcha] - ${request}`); | |||
return false; | |||
} | |||
return Number(request) > 0; | |||
} catch (error) { | |||
console.log(`[consultar-saldo-captcha] - ${error.message}`); | |||
return false; | |||
} | |||
} | |||
private async _sendCaptcha (imageData: string): Promise<string> { | |||
try { | |||
const captchaResponse = await axios.post( | |||
this.CAPTCHA_CREATE, | |||
{ | |||
method: 'base64', | |||
key: this.CAPTCHA_KEY, | |||
body: imageData, | |||
json: 1 | |||
}, | |||
{ headers: { 'Content-Type': 'multipart/form-data' } } | |||
) | |||
let response: string = '' | |||
if (captchaResponse.data && typeof captchaResponse.data === 'object') { | |||
if (captchaResponse.data.status === 0) | |||
throw new Error(captchaResponse.data.request) | |||
response = captchaResponse.data.request | |||
} else if ( | |||
captchaResponse.data && | |||
typeof captchaResponse.data === 'string' | |||
) { | |||
if (captchaResponse.data.indexOf('OK') === -1) | |||
throw new Error(captchaResponse.data) | |||
response = captchaResponse.data.split('|')[1] | |||
} | |||
return response | |||
} catch (error) { | |||
return new Promise((resolve, reject) => reject(error)) | |||
} | |||
} | |||
private async _getResponseCaptcha (captchaId: string): Promise<string> { | |||
return new Promise((resolve, reject) => { | |||
const interval = setInterval(async () => { | |||
try { | |||
const serverResponse = await axios.get(this.CAPTCHA_RESULT, { | |||
params: { | |||
key: this.CAPTCHA_KEY, | |||
action: 'get', | |||
id: captchaId, | |||
json: 1 | |||
} | |||
}) | |||
if (serverResponse.data.request !== 'CAPCHA_NOT_READY') { | |||
clearInterval(interval) | |||
let response = '' | |||
if ( | |||
serverResponse.data && | |||
typeof serverResponse.data === 'object' | |||
) { | |||
if (serverResponse.data.status === 0) | |||
throw new Error(serverResponse.data.request) | |||
response = serverResponse.data.request | |||
} else if ( | |||
serverResponse.data && | |||
typeof serverResponse.data === 'string' | |||
) { | |||
if (serverResponse.data.indexOf('OK') === -1) | |||
throw new Error(serverResponse.data) | |||
response = serverResponse.data.split('|')[1] | |||
} | |||
resolve(response) | |||
} | |||
} catch (e) { | |||
reject(e) | |||
} | |||
}, 10 * 1000) | |||
}) | |||
} | |||
} |
@@ -0,0 +1,4 @@ | |||
export interface ICaptchaResolver { | |||
temSaldoDisponivel(): Promise<boolean>; | |||
resolver(imageData: string): Promise<string>; | |||
} |
@@ -0,0 +1,6 @@ | |||
export interface CertidaoNegativa { | |||
situacao: number; | |||
dataEmissao: number; | |||
dataValidade?: number; | |||
file?: Uint8Array; | |||
} |
@@ -0,0 +1,2 @@ | |||
export * from './certidao-negativa.interface'; | |||
export * from './scraper-data.interface'; |
@@ -0,0 +1,4 @@ | |||
export interface ScrapeData { | |||
inscricao: string; | |||
antecipar?: boolean; | |||
} |
@@ -0,0 +1,7 @@ | |||
import { NestFactory } from '@nestjs/core'; | |||
import { AppModule } from './app.module'; | |||
async function bootstrap() { | |||
await NestFactory.createApplicationContext(AppModule); | |||
} | |||
bootstrap(); |
@@ -0,0 +1,26 @@ | |||
import { BullModule } from '@nestjs/bull'; | |||
import { Module } from '@nestjs/common'; | |||
import { ConfigModule, ConfigService } from '@nestjs/config'; | |||
import { ScraperMtProcessor } from './scraper-cnd-mt.processor'; | |||
import { ScraperCndMtServise } from './scraper-cnd-mt.service'; | |||
@Module({ | |||
imports: [ | |||
BullModule.registerQueueAsync({ | |||
name: 'mato-grosso', | |||
imports: [ConfigModule], | |||
useFactory: async (configService: ConfigService) => ({ | |||
redis: { | |||
host: configService.get('APP_REDIS_HOST'), | |||
port: +configService.get('APP_REDIS_PORT'), | |||
} | |||
}), | |||
inject: [ConfigService], | |||
})], | |||
providers: [ | |||
ScraperCndMtServise, | |||
ScraperMtProcessor | |||
], | |||
}) | |||
export class ScraperCndMtModule {} |
@@ -0,0 +1,15 @@ | |||
import { Process, Processor } from "@nestjs/bull"; | |||
import { Job } from 'bull'; | |||
import { ScraperCndMtServise } from "./scraper-cnd-mt.service"; | |||
@Processor('mato-grosso') | |||
export class ScraperMtProcessor { | |||
constructor(private readonly scraperCndMtService: ScraperCndMtServise) {} | |||
@Process({ name: 'mato-grosso', concurrency: 1}) | |||
handler(job: Job) { | |||
const { inscricao } = job.data; | |||
return this.scraperCndMtService.scraperCndMt(inscricao); | |||
} | |||
} |
@@ -0,0 +1,265 @@ | |||
import { Injectable, Logger, Scope } from '@nestjs/common'; | |||
import puppeteer, { Browser, Page } from 'puppeteer'; | |||
import pdfparse from 'pdf-parse'; | |||
import { getUnixTime } from 'date-fns'; | |||
import { certidaoResult } from '../../utils/certidao.utils'; | |||
import { Situacao } from 'src/enums/situacao.enum'; | |||
import { CapMonster } from 'src/helpers/capMonster'; | |||
import { TwoCaptcha } from 'src/helpers/twoCaptcha'; | |||
import { CertidaoNegativa } from 'src/interfaces'; | |||
@Injectable({ scope: Scope.REQUEST }) | |||
export class ScraperCndMtServise { | |||
private _page: Page; | |||
private browser: Browser; | |||
private cndFile: Buffer; | |||
private resultadoScraping: CertidaoNegativa; | |||
private logger = new Logger('ScraperCndMtServise'); | |||
constructor() { } | |||
async scraperCndMt(inscricao: string) { | |||
try { | |||
await this.paginaInicial(inscricao); | |||
if(this.cndFile) { | |||
await this.capturandoInformacoesPdf(this.cndFile); | |||
} else { | |||
const selecionandoTabela = await this._page.$$('table'); | |||
const selecionandoBodyTabela = await selecionandoTabela[2].$$('tbody > tr'); | |||
const selecionandoConteudoDiv = await selecionandoBodyTabela[2].$$('td > div'); | |||
const mensagemCndPositiva = await selecionandoConteudoDiv[0].$eval( | |||
'font' , | |||
elem => String(elem.textContent).trim(), | |||
); | |||
if(mensagemCndPositiva.includes('não são suficientes')) { | |||
this.logger.log('Certidão Positiva'); | |||
return certidaoResult(Situacao.Positiva) | |||
} | |||
} | |||
await this._page.waitFor(8000) | |||
console.log(this.resultadoScraping) | |||
return this.resultadoScraping; | |||
} catch (error) { | |||
this.logger.error(error.message); | |||
return certidaoResult(Situacao.FalhaConsulta); | |||
} finally { | |||
await this._page.close(); | |||
await this.browser.close(); | |||
} | |||
} | |||
private async paginaInicial(inscricao: string): Promise<any> { | |||
let tentativas = 0; | |||
do { | |||
await this.carregarNavegadorPaginaInicial(); | |||
this.logger.log(`Gerando nova CND para a inscrição ${inscricao}...`); | |||
await this.preencheFormulario(inscricao); | |||
await this.resolveCaptcha(); | |||
await this._page.waitForNavigation(); | |||
const validandoResolucaoDoCaptcha = await this._page.$$eval('form > font > b', mensagem => mensagem.map(texto => texto.textContent)) | |||
if(validandoResolucaoDoCaptcha[0]) { | |||
if(validandoResolucaoDoCaptcha[0].includes('Código de caracteres inválido')){ | |||
this.logger.log(`Captcha incorreto!`); | |||
tentativas += 1; | |||
await this.browser.close(); | |||
} | |||
} | |||
if(!validandoResolucaoDoCaptcha[0]){ | |||
this.logger.log(`Captcha para a inscrição ${inscricao} foi resolvido`); | |||
await this.capturaArquivoNaRequisicao(); | |||
break; | |||
} | |||
} while (tentativas != 2); | |||
} | |||
private async carregarNavegadorPaginaInicial() { | |||
this.browser = await puppeteer.launch({ | |||
headless: false, | |||
defaultViewport: null, | |||
}); | |||
this._page = await this.browser.newPage(); | |||
await this._page.goto( | |||
'https://www.sefaz.mt.gov.br/cnd/certidao/servlet/ServletRotd?origem=60', | |||
{ | |||
waitUntil: 'load', | |||
} | |||
) | |||
return this._page; | |||
} | |||
private async preencheFormulario(inscricao: string) { | |||
await this._page.waitForSelector('#ModeloCertidao'); | |||
await this._page.click('#ModeloCertidao'); | |||
await this._page.waitForSelector('#tipoDoct'); | |||
const selecionandoOpcaoCnpj = await this._page.$$('#tipoDoct'); | |||
await selecionandoOpcaoCnpj[1].click(); | |||
await this._page.waitForSelector('#numrDoctCNPJ'); | |||
await this._page.type("#numrDoctCNPJ", inscricao, { delay: 250 }); | |||
} | |||
private async capturaArquivoNaRequisicao (): Promise<any> { | |||
const client = await this._page.target().createCDPSession(); | |||
let pdfBase64: Array<string> = []; | |||
let responseObj: Object; | |||
client.send('Fetch.enable', { | |||
patterns: [ | |||
{ | |||
requestStage: 'Response', | |||
}, | |||
], | |||
}) | |||
await client.on('Fetch.requestPaused', async (reqEvent) => { | |||
const { requestId } = reqEvent; | |||
this.logger.log('Requisição pausada') | |||
let responseHeaders = reqEvent.responseHeaders || []; | |||
let contentType = ''; | |||
for (let elements of responseHeaders) { | |||
if (elements.name.toLowerCase() === 'content-type') { | |||
contentType = elements.value; | |||
} | |||
} | |||
if (contentType.endsWith('pdf')) { | |||
const foundHeaderIndex = responseHeaders.findIndex( | |||
(h) => h.name === "content-disposition" | |||
); | |||
const attachmentHeader = { | |||
name: "content-disposition", | |||
value: "attachment", | |||
}; | |||
if (foundHeaderIndex) { | |||
responseHeaders[foundHeaderIndex] = attachmentHeader; | |||
} else { | |||
responseHeaders.push(attachmentHeader); | |||
} | |||
responseObj = await client.send('Fetch.getResponseBody', { | |||
requestId, | |||
}); | |||
pdfBase64 = Object.values(responseObj); | |||
this.cndFile = new Buffer(pdfBase64[0], 'base64'); | |||
if(this.cndFile){ | |||
this.logger.log('Cnd capturada') | |||
} | |||
await client.send('Fetch.continueRequest', { requestId }); | |||
} else { | |||
await client.send('Fetch.continueRequest', { requestId }); | |||
} | |||
}); | |||
await this._page.waitFor(25000) | |||
} | |||
private async capturandoInformacoesPdf(cndFile: Buffer): Promise<any> { | |||
let situacao = 0; | |||
this.logger.log('Extraindo informações do pdf') | |||
let data = await pdfparse(cndFile) | |||
if(data.text.includes('CERTIDÃO NEGATIVA')){ | |||
situacao = 1; | |||
} else if(data.text.includes('CERTIDÃO POSITIVA COM EFEITOS DE NEGATIVA')) { | |||
situacao = 4; | |||
} | |||
let inicioSeletor = data.text.indexOf('válida até:'); | |||
const posicaoInicialDataValidade = inicioSeletor + 13; | |||
const posicaoFinalDataValidade = posicaoInicialDataValidade + 10; | |||
const dataValidade = new Date(data.text.slice(posicaoInicialDataValidade, posicaoFinalDataValidade)); | |||
let inicioSeletorDataEmissão = data.text.indexOf('Data da emissão:'); | |||
const posicaoInicialDataEmissao = inicioSeletorDataEmissão + 18; | |||
const posicaoFinalDataEmissao = posicaoInicialDataEmissao + 10; | |||
const dataEmissao = new Date(data.text.slice(posicaoInicialDataEmissao, posicaoFinalDataEmissao)); | |||
if(dataEmissao && dataValidade && cndFile) { | |||
this.logger.log('Scraping concluído'); | |||
this.resultadoScraping = { | |||
situacao: situacao, | |||
dataEmissao: getUnixTime(dataEmissao), | |||
dataValidade: getUnixTime(dataValidade), | |||
file: cndFile | |||
} | |||
} | |||
} | |||
private async resolveCaptcha (): Promise<any> { | |||
const captchaBinary = await this._page.evaluate(() => { | |||
const img: HTMLImageElement = document.querySelector( | |||
'[src="/cnd/certidao/geradorcaracteres"]' | |||
) | |||
const canvas = document.createElement('canvas') | |||
canvas.width = img.width | |||
canvas.height = img.height | |||
const ctx = canvas.getContext('2d') | |||
ctx.drawImage(img, 0, 0) | |||
return canvas.toDataURL('image/png') | |||
}) | |||
const captchaService = new CapMonster() | |||
const twoCaptchService = new TwoCaptcha() | |||
const matches = captchaBinary.match(/^data:(.+);base64,(.+)$/) | |||
if (matches.length !== 3) { | |||
throw new Error('Erro ao extrair imagem do Captcha') | |||
} | |||
let captcha; | |||
if (await captchaService.temSaldoDisponivel()) { | |||
this.logger.log(' Usando CapMonster...') | |||
captcha = await captchaService.resolver(matches[2]); | |||
} else if (await twoCaptchService.temSaldoDisponivel()) { | |||
this.logger.log(' Usando TwoCaptcha...') | |||
captcha = await twoCaptchService.resolveCaptcha(matches[2]); | |||
} else { | |||
this.logger.log('Não tem saldo para quebrar o captcha.'); | |||
return { | |||
sucesso: false, | |||
mensagem: 'Não tem saldo para quebrar o captcha.' | |||
} | |||
} | |||
if (!captcha) throw new Error('Erro ao resolver o Captcha'); | |||
await this._page.type('[name="caracteres"]', captcha, {delay: 250}); | |||
const selecionandoElementoBotaoOk = await this._page.$$('#spanBotao'); | |||
await selecionandoElementoBotaoOk[0].click(); | |||
} | |||
} |
@@ -0,0 +1 @@ | |||
declare module '@infosimples/node_two_captcha'; |
@@ -0,0 +1,4 @@ | |||
{ | |||
"extends": "./tsconfig.json", | |||
"exclude": ["node_modules", "test", "dist", "**/*spec.ts"] | |||
} |
@@ -0,0 +1,16 @@ | |||
{ | |||
"compilerOptions": { | |||
"module": "commonjs", | |||
"declaration": true, | |||
"removeComments": true, | |||
"emitDecoratorMetadata": true, | |||
"experimentalDecorators": true, | |||
"target": "es2017", | |||
"sourceMap": true, | |||
"outDir": "./dist", | |||
"baseUrl": "./", | |||
"incremental": true, | |||
"esModuleInterop": true | |||
}, | |||
"exclude": ["node_modules", "dist"] | |||
} |