| @@ -0,0 +1,5 @@ | |||
| APP_REDIS_HOST=redis-cnds | |||
| APP_REDIS_PORT=6380 | |||
| CAP_MONSTER_KEY =89310c8388da93bf17133bf399f517de | |||
| TWO_CAPTCHA_KEY=0a26ea85b64ea9797cc296337001f6ab | |||
| @@ -0,0 +1,24 @@ | |||
| module.exports = { | |||
| parser: '@typescript-eslint/parser', | |||
| parserOptions: { | |||
| project: 'tsconfig.json', | |||
| sourceType: 'module', | |||
| }, | |||
| plugins: ['@typescript-eslint/eslint-plugin'], | |||
| extends: [ | |||
| 'plugin:@typescript-eslint/recommended', | |||
| 'plugin:prettier/recommended', | |||
| ], | |||
| root: true, | |||
| env: { | |||
| node: true, | |||
| jest: true, | |||
| }, | |||
| ignorePatterns: ['.eslintrc.js'], | |||
| rules: { | |||
| '@typescript-eslint/interface-name-prefix': 'off', | |||
| '@typescript-eslint/explicit-function-return-type': 'off', | |||
| '@typescript-eslint/explicit-module-boundary-types': 'off', | |||
| '@typescript-eslint/no-explicit-any': 'off', | |||
| }, | |||
| }; | |||
| @@ -0,0 +1,35 @@ | |||
| # compiled output | |||
| /dist | |||
| /node_modules | |||
| # Logs | |||
| logs | |||
| *.log | |||
| npm-debug.log* | |||
| pnpm-debug.log* | |||
| yarn-debug.log* | |||
| yarn-error.log* | |||
| lerna-debug.log* | |||
| # OS | |||
| .DS_Store | |||
| # Tests | |||
| /coverage | |||
| /.nyc_output | |||
| # IDEs and editors | |||
| /.idea | |||
| .project | |||
| .classpath | |||
| .c9/ | |||
| *.launch | |||
| .settings/ | |||
| *.sublime-workspace | |||
| # IDE - VSCode | |||
| .vscode/* | |||
| !.vscode/settings.json | |||
| !.vscode/tasks.json | |||
| !.vscode/launch.json | |||
| !.vscode/extensions.json | |||
| @@ -0,0 +1,4 @@ | |||
| { | |||
| "singleQuote": true, | |||
| "trailingComma": "all" | |||
| } | |||
| @@ -0,0 +1,73 @@ | |||
| <p align="center"> | |||
| <a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo_text.svg" width="320" alt="Nest Logo" /></a> | |||
| </p> | |||
| [circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456 | |||
| [circleci-url]: https://circleci.com/gh/nestjs/nest | |||
| <p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p> | |||
| <p align="center"> | |||
| <a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a> | |||
| <a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a> | |||
| <a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a> | |||
| <a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a> | |||
| <a href="https://coveralls.io/github/nestjs/nest?branch=master" target="_blank"><img src="https://coveralls.io/repos/github/nestjs/nest/badge.svg?branch=master#9" alt="Coverage" /></a> | |||
| <a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a> | |||
| <a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a> | |||
| <a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a> | |||
| <a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg"/></a> | |||
| <a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a> | |||
| <a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow"></a> | |||
| </p> | |||
| <!--[](https://opencollective.com/nest#backer) | |||
| [](https://opencollective.com/nest#sponsor)--> | |||
| ## Description | |||
| [Nest](https://github.com/nestjs/nest) framework TypeScript starter repository. | |||
| ## Installation | |||
| ```bash | |||
| $ npm install | |||
| ``` | |||
| ## Running the app | |||
| ```bash | |||
| # development | |||
| $ npm run start | |||
| # watch mode | |||
| $ npm run start:dev | |||
| # production mode | |||
| $ npm run start:prod | |||
| ``` | |||
| ## Test | |||
| ```bash | |||
| # unit tests | |||
| $ npm run test | |||
| # e2e tests | |||
| $ npm run test:e2e | |||
| # test coverage | |||
| $ npm run test:cov | |||
| ``` | |||
| ## Support | |||
| Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support). | |||
| ## Stay in touch | |||
| - Author - [Kamil Myśliwiec](https://kamilmysliwiec.com) | |||
| - Website - [https://nestjs.com](https://nestjs.com/) | |||
| - Twitter - [@nestframework](https://twitter.com/nestframework) | |||
| ## License | |||
| Nest is [MIT licensed](LICENSE). | |||
| @@ -0,0 +1,4 @@ | |||
| { | |||
| "collection": "@nestjs/schematics", | |||
| "sourceRoot": "src" | |||
| } | |||
| @@ -0,0 +1,82 @@ | |||
| { | |||
| "name": "scraper-cnd-mt", | |||
| "version": "0.0.1", | |||
| "description": "", | |||
| "author": "", | |||
| "private": true, | |||
| "license": "UNLICENSED", | |||
| "scripts": { | |||
| "prebuild": "rimraf dist", | |||
| "build": "nest build", | |||
| "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", | |||
| "start": "nest start", | |||
| "start:dev": "nest start --watch", | |||
| "start:debug": "nest start --debug --watch", | |||
| "start:prod": "node dist/main", | |||
| "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", | |||
| "test": "jest", | |||
| "test:watch": "jest --watch", | |||
| "test:cov": "jest --coverage", | |||
| "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", | |||
| "test:e2e": "jest --config ./test/jest-e2e.json", | |||
| "docker:build": "", | |||
| "docker:tag": "", | |||
| "docker:push": "", | |||
| "docker:publish": "npm run docker:tag && npm run docker:push" | |||
| }, | |||
| "dependencies": { | |||
| "@infosimples/node_two_captcha": "^1.0.2", | |||
| "@nestjs/bull": "^0.1.2", | |||
| "@nestjs/common": "^7.6.18", | |||
| "@nestjs/config": "^0.4.0", | |||
| "@nestjs/core": "^7.0.0", | |||
| "@nestjs/platform-express": "^7.0.0", | |||
| "bull": "^3.15.0", | |||
| "data-fns": "^0.1.8", | |||
| "date-fns": "^2.12.0", | |||
| "pdf-parse": "^1.1.1", | |||
| "puppeteer": "^5.0.0", | |||
| "reflect-metadata": "^0.1.13", | |||
| "rimraf": "^3.0.2", | |||
| "rxjs": "^6.5.4" | |||
| }, | |||
| "devDependencies": { | |||
| "@nestjs/cli": "^7.0.0", | |||
| "@nestjs/schematics": "^7.0.0", | |||
| "@nestjs/testing": "^7.0.0", | |||
| "@types/bull": "^3.14.0", | |||
| "@types/express": "^4.17.3", | |||
| "@types/jest": "25.1.4", | |||
| "@types/node": "^13.9.1", | |||
| "@types/pdf-parse": "^1.1.1", | |||
| "@types/puppeteer": "^3.0.1", | |||
| "@types/supertest": "^2.0.8", | |||
| "@typescript-eslint/eslint-plugin": "^2.23.0", | |||
| "@typescript-eslint/parser": "^2.23.0", | |||
| "eslint": "^6.8.0", | |||
| "eslint-config-prettier": "^6.10.0", | |||
| "eslint-plugin-import": "^2.20.1", | |||
| "jest": "^25.1.0", | |||
| "prettier": "^1.19.1", | |||
| "supertest": "^4.0.2", | |||
| "ts-jest": "25.2.1", | |||
| "ts-loader": "^6.2.1", | |||
| "ts-node": "^8.6.2", | |||
| "tsconfig-paths": "^3.9.0", | |||
| "typescript": "^3.7.4" | |||
| }, | |||
| "jest": { | |||
| "moduleFileExtensions": [ | |||
| "js", | |||
| "json", | |||
| "ts" | |||
| ], | |||
| "rootDir": "src", | |||
| "testRegex": ".spec.ts$", | |||
| "transform": { | |||
| "^.+\\.(t|j)s$": "ts-jest" | |||
| }, | |||
| "coverageDirectory": "../coverage", | |||
| "testEnvironment": "node" | |||
| } | |||
| } | |||
| @@ -0,0 +1,12 @@ | |||
| import { Module } from '@nestjs/common'; | |||
| import { ConfigModule } from '@nestjs/config'; | |||
| import { ScraperCndMtModule } from './modules/scraper-cnd-mt/scraper-cnd-mt.module'; | |||
| @Module({ | |||
| imports: [ | |||
| ConfigModule.forRoot({ isGlobal: true }), | |||
| ScraperCndMtModule | |||
| ], | |||
| }) | |||
| export class AppModule {} | |||
| @@ -0,0 +1,6 @@ | |||
| export enum Situacao { | |||
| Negativa = 1, | |||
| Positiva, | |||
| FalhaConsulta, | |||
| PositivaEfeitoNegativa, | |||
| } | |||
| @@ -0,0 +1,114 @@ | |||
| import axios from 'axios' | |||
| import { ICaptchaResolver } from '../interfaces/ICaptchaResolver'; | |||
| export class CapMonster implements ICaptchaResolver { | |||
| private readonly CAPTCHA_KEY: string = process.env.CAP_MONSTER_KEY | |||
| private readonly CAPTCHA_BALANCE: string = | |||
| 'https://api.capmonster.cloud/getBalance' | |||
| private readonly CAPTCHA_CREATE: string = | |||
| 'https://api.capmonster.cloud/createTask' | |||
| private readonly CAPTCHA_RESULT: string = | |||
| 'https://api.capmonster.cloud/getTaskResult' | |||
| public async temSaldoDisponivel (): Promise<boolean> { | |||
| try { | |||
| const balanceData = { | |||
| clientKey: this.CAPTCHA_KEY | |||
| } | |||
| const { data } = await axios.post(this.CAPTCHA_BALANCE, balanceData) | |||
| if (data.errorId > 0) { | |||
| console.log(`[consultar-saldo-captcha] - ${data.errorCode}`) | |||
| return false | |||
| } | |||
| return data.balance > 0 | |||
| } catch (error) { | |||
| console.log(`[consultar-saldo-captcha] - ${error.message}`) | |||
| return false | |||
| } | |||
| } | |||
| public async resolver (imageData: string): Promise<string> { | |||
| const taskId = await this.criar(imageData) | |||
| if (taskId === 0) return null | |||
| return await this.resultado(taskId) | |||
| } | |||
| private async criar (imageData: string): Promise<number> { | |||
| try { | |||
| const createTask = { | |||
| clientKey: this.CAPTCHA_KEY, | |||
| task: { | |||
| type: 'ImageToTextTask', | |||
| body: imageData | |||
| } | |||
| } | |||
| const { data } = await axios.post(this.CAPTCHA_CREATE, createTask) | |||
| if (data.errorId > 0) { | |||
| console.log(`[criar-captcha] - ${data.errorCode}`) | |||
| return 0 | |||
| } | |||
| return data.taskId | |||
| } catch (error) { | |||
| console.log(`[criar-captcha] - ${error.message}`) | |||
| return 0 | |||
| } | |||
| } | |||
| private async resultado (taskId: number): Promise<string> { | |||
| let tentativas = 0 | |||
| let error: boolean = false | |||
| let captchaData = null | |||
| const getResultTask = { | |||
| clientKey: this.CAPTCHA_KEY, | |||
| taskId: taskId | |||
| } | |||
| do { | |||
| try { | |||
| const { | |||
| data: { errorId, errorCode, solution, status } | |||
| } = await axios.post(this.CAPTCHA_RESULT, getResultTask) | |||
| if (errorId > 0) { | |||
| console.log( | |||
| `[resultado-captcha] - Task: ${taskId} Erro: ${errorCode} Tentativas: ${tentativas}` | |||
| ) | |||
| error = true | |||
| break | |||
| } | |||
| if (status === 'ready') { | |||
| captchaData = solution.text | |||
| break | |||
| } | |||
| tentativas += 1 | |||
| await new Promise(r => setTimeout(r, 10 * 1000)) | |||
| } catch (error) { | |||
| console.log( | |||
| `[resultado-captcha] - Task: ${taskId} Erro: ${error.message} Tentativas: ${tentativas}` | |||
| ) | |||
| error = true | |||
| break | |||
| } | |||
| } while (tentativas < 20) | |||
| if (error) return null | |||
| return captchaData | |||
| } | |||
| } | |||
| export default new CapMonster() | |||
| @@ -0,0 +1,114 @@ | |||
| import axios from 'axios' | |||
| export class TwoCaptcha { | |||
| private readonly CAPTCHA_KEY: string = 'TWO_CAPTCHA_KEY' | |||
| private readonly CAPTCHA_CREATE: string = 'http://2captcha.com/in.php' | |||
| private readonly CAPTCHA_RESULT: string = 'http://2captcha.com/res.php' | |||
| private readonly CAPTCHA_BALANCE: string = 'https://2captcha.com/res.php' | |||
| async resolveCaptcha (imageData: string) { | |||
| const captchaId = await this._sendCaptcha(imageData) | |||
| const captchaResolved = await this._getResponseCaptcha(captchaId) | |||
| return captchaResolved | |||
| } | |||
| public async temSaldoDisponivel (): Promise<boolean> { | |||
| try { | |||
| const balanceData = { | |||
| key: this.CAPTCHA_KEY, | |||
| action: 'getbalance', | |||
| json: 1 | |||
| } | |||
| const { data: { request } } = await axios.get(this.CAPTCHA_BALANCE, { params: balanceData }); | |||
| if ((request || '').includes('ERROR_')) { | |||
| console.log(`[consultar-saldo-captcha] - ${request}`); | |||
| return false; | |||
| } | |||
| return Number(request) > 0; | |||
| } catch (error) { | |||
| console.log(`[consultar-saldo-captcha] - ${error.message}`); | |||
| return false; | |||
| } | |||
| } | |||
| private async _sendCaptcha (imageData: string): Promise<string> { | |||
| try { | |||
| const captchaResponse = await axios.post( | |||
| this.CAPTCHA_CREATE, | |||
| { | |||
| method: 'base64', | |||
| key: this.CAPTCHA_KEY, | |||
| body: imageData, | |||
| json: 1 | |||
| }, | |||
| { headers: { 'Content-Type': 'multipart/form-data' } } | |||
| ) | |||
| let response: string = '' | |||
| if (captchaResponse.data && typeof captchaResponse.data === 'object') { | |||
| if (captchaResponse.data.status === 0) | |||
| throw new Error(captchaResponse.data.request) | |||
| response = captchaResponse.data.request | |||
| } else if ( | |||
| captchaResponse.data && | |||
| typeof captchaResponse.data === 'string' | |||
| ) { | |||
| if (captchaResponse.data.indexOf('OK') === -1) | |||
| throw new Error(captchaResponse.data) | |||
| response = captchaResponse.data.split('|')[1] | |||
| } | |||
| return response | |||
| } catch (error) { | |||
| return new Promise((resolve, reject) => reject(error)) | |||
| } | |||
| } | |||
| private async _getResponseCaptcha (captchaId: string): Promise<string> { | |||
| return new Promise((resolve, reject) => { | |||
| const interval = setInterval(async () => { | |||
| try { | |||
| const serverResponse = await axios.get(this.CAPTCHA_RESULT, { | |||
| params: { | |||
| key: this.CAPTCHA_KEY, | |||
| action: 'get', | |||
| id: captchaId, | |||
| json: 1 | |||
| } | |||
| }) | |||
| if (serverResponse.data.request !== 'CAPCHA_NOT_READY') { | |||
| clearInterval(interval) | |||
| let response = '' | |||
| if ( | |||
| serverResponse.data && | |||
| typeof serverResponse.data === 'object' | |||
| ) { | |||
| if (serverResponse.data.status === 0) | |||
| throw new Error(serverResponse.data.request) | |||
| response = serverResponse.data.request | |||
| } else if ( | |||
| serverResponse.data && | |||
| typeof serverResponse.data === 'string' | |||
| ) { | |||
| if (serverResponse.data.indexOf('OK') === -1) | |||
| throw new Error(serverResponse.data) | |||
| response = serverResponse.data.split('|')[1] | |||
| } | |||
| resolve(response) | |||
| } | |||
| } catch (e) { | |||
| reject(e) | |||
| } | |||
| }, 10 * 1000) | |||
| }) | |||
| } | |||
| } | |||
| @@ -0,0 +1,4 @@ | |||
| export interface ICaptchaResolver { | |||
| temSaldoDisponivel(): Promise<boolean>; | |||
| resolver(imageData: string): Promise<string>; | |||
| } | |||
| @@ -0,0 +1,6 @@ | |||
| export interface CertidaoNegativa { | |||
| situacao: number; | |||
| dataEmissao: number; | |||
| dataValidade?: number; | |||
| file?: Uint8Array; | |||
| } | |||
| @@ -0,0 +1,2 @@ | |||
| export * from './certidao-negativa.interface'; | |||
| export * from './scraper-data.interface'; | |||
| @@ -0,0 +1,4 @@ | |||
| export interface ScrapeData { | |||
| inscricao: string; | |||
| antecipar?: boolean; | |||
| } | |||
| @@ -0,0 +1,7 @@ | |||
| import { NestFactory } from '@nestjs/core'; | |||
| import { AppModule } from './app.module'; | |||
| async function bootstrap() { | |||
| await NestFactory.createApplicationContext(AppModule); | |||
| } | |||
| bootstrap(); | |||
| @@ -0,0 +1,26 @@ | |||
| import { BullModule } from '@nestjs/bull'; | |||
| import { Module } from '@nestjs/common'; | |||
| import { ConfigModule, ConfigService } from '@nestjs/config'; | |||
| import { ScraperMtProcessor } from './scraper-cnd-mt.processor'; | |||
| import { ScraperCndMtServise } from './scraper-cnd-mt.service'; | |||
| @Module({ | |||
| imports: [ | |||
| BullModule.registerQueueAsync({ | |||
| name: 'mato-grosso', | |||
| imports: [ConfigModule], | |||
| useFactory: async (configService: ConfigService) => ({ | |||
| redis: { | |||
| host: configService.get('APP_REDIS_HOST'), | |||
| port: +configService.get('APP_REDIS_PORT'), | |||
| } | |||
| }), | |||
| inject: [ConfigService], | |||
| })], | |||
| providers: [ | |||
| ScraperCndMtServise, | |||
| ScraperMtProcessor | |||
| ], | |||
| }) | |||
| export class ScraperCndMtModule {} | |||
| @@ -0,0 +1,15 @@ | |||
| import { Process, Processor } from "@nestjs/bull"; | |||
| import { Job } from 'bull'; | |||
| import { ScraperCndMtServise } from "./scraper-cnd-mt.service"; | |||
| @Processor('mato-grosso') | |||
| export class ScraperMtProcessor { | |||
| constructor(private readonly scraperCndMtService: ScraperCndMtServise) {} | |||
| @Process({ name: 'mato-grosso', concurrency: 1}) | |||
| handler(job: Job) { | |||
| const { inscricao } = job.data; | |||
| return this.scraperCndMtService.scraperCndMt(inscricao); | |||
| } | |||
| } | |||
| @@ -0,0 +1,265 @@ | |||
| import { Injectable, Logger, Scope } from '@nestjs/common'; | |||
| import puppeteer, { Browser, Page } from 'puppeteer'; | |||
| import pdfparse from 'pdf-parse'; | |||
| import { getUnixTime } from 'date-fns'; | |||
| import { certidaoResult } from '../../utils/certidao.utils'; | |||
| import { Situacao } from 'src/enums/situacao.enum'; | |||
| import { CapMonster } from 'src/helpers/capMonster'; | |||
| import { TwoCaptcha } from 'src/helpers/twoCaptcha'; | |||
| import { CertidaoNegativa } from 'src/interfaces'; | |||
| @Injectable({ scope: Scope.REQUEST }) | |||
| export class ScraperCndMtServise { | |||
| private _page: Page; | |||
| private browser: Browser; | |||
| private cndFile: Buffer; | |||
| private resultadoScraping: CertidaoNegativa; | |||
| private logger = new Logger('ScraperCndMtServise'); | |||
| constructor() { } | |||
| async scraperCndMt(inscricao: string) { | |||
| try { | |||
| await this.paginaInicial(inscricao); | |||
| if(this.cndFile) { | |||
| await this.capturandoInformacoesPdf(this.cndFile); | |||
| } else { | |||
| const selecionandoTabela = await this._page.$$('table'); | |||
| const selecionandoBodyTabela = await selecionandoTabela[2].$$('tbody > tr'); | |||
| const selecionandoConteudoDiv = await selecionandoBodyTabela[2].$$('td > div'); | |||
| const mensagemCndPositiva = await selecionandoConteudoDiv[0].$eval( | |||
| 'font' , | |||
| elem => String(elem.textContent).trim(), | |||
| ); | |||
| if(mensagemCndPositiva.includes('não são suficientes')) { | |||
| this.logger.log('Certidão Positiva'); | |||
| return certidaoResult(Situacao.Positiva) | |||
| } | |||
| } | |||
| await this._page.waitFor(8000) | |||
| console.log(this.resultadoScraping) | |||
| return this.resultadoScraping; | |||
| } catch (error) { | |||
| this.logger.error(error.message); | |||
| return certidaoResult(Situacao.FalhaConsulta); | |||
| } finally { | |||
| await this._page.close(); | |||
| await this.browser.close(); | |||
| } | |||
| } | |||
| private async paginaInicial(inscricao: string): Promise<any> { | |||
| let tentativas = 0; | |||
| do { | |||
| await this.carregarNavegadorPaginaInicial(); | |||
| this.logger.log(`Gerando nova CND para a inscrição ${inscricao}...`); | |||
| await this.preencheFormulario(inscricao); | |||
| await this.resolveCaptcha(); | |||
| await this._page.waitForNavigation(); | |||
| const validandoResolucaoDoCaptcha = await this._page.$$eval('form > font > b', mensagem => mensagem.map(texto => texto.textContent)) | |||
| if(validandoResolucaoDoCaptcha[0]) { | |||
| if(validandoResolucaoDoCaptcha[0].includes('Código de caracteres inválido')){ | |||
| this.logger.log(`Captcha incorreto!`); | |||
| tentativas += 1; | |||
| await this.browser.close(); | |||
| } | |||
| } | |||
| if(!validandoResolucaoDoCaptcha[0]){ | |||
| this.logger.log(`Captcha para a inscrição ${inscricao} foi resolvido`); | |||
| await this.capturaArquivoNaRequisicao(); | |||
| break; | |||
| } | |||
| } while (tentativas != 2); | |||
| } | |||
| private async carregarNavegadorPaginaInicial() { | |||
| this.browser = await puppeteer.launch({ | |||
| headless: false, | |||
| defaultViewport: null, | |||
| }); | |||
| this._page = await this.browser.newPage(); | |||
| await this._page.goto( | |||
| 'https://www.sefaz.mt.gov.br/cnd/certidao/servlet/ServletRotd?origem=60', | |||
| { | |||
| waitUntil: 'load', | |||
| } | |||
| ) | |||
| return this._page; | |||
| } | |||
| private async preencheFormulario(inscricao: string) { | |||
| await this._page.waitForSelector('#ModeloCertidao'); | |||
| await this._page.click('#ModeloCertidao'); | |||
| await this._page.waitForSelector('#tipoDoct'); | |||
| const selecionandoOpcaoCnpj = await this._page.$$('#tipoDoct'); | |||
| await selecionandoOpcaoCnpj[1].click(); | |||
| await this._page.waitForSelector('#numrDoctCNPJ'); | |||
| await this._page.type("#numrDoctCNPJ", inscricao, { delay: 250 }); | |||
| } | |||
| private async capturaArquivoNaRequisicao (): Promise<any> { | |||
| const client = await this._page.target().createCDPSession(); | |||
| let pdfBase64: Array<string> = []; | |||
| let responseObj: Object; | |||
| client.send('Fetch.enable', { | |||
| patterns: [ | |||
| { | |||
| requestStage: 'Response', | |||
| }, | |||
| ], | |||
| }) | |||
| await client.on('Fetch.requestPaused', async (reqEvent) => { | |||
| const { requestId } = reqEvent; | |||
| this.logger.log('Requisição pausada') | |||
| let responseHeaders = reqEvent.responseHeaders || []; | |||
| let contentType = ''; | |||
| for (let elements of responseHeaders) { | |||
| if (elements.name.toLowerCase() === 'content-type') { | |||
| contentType = elements.value; | |||
| } | |||
| } | |||
| if (contentType.endsWith('pdf')) { | |||
| const foundHeaderIndex = responseHeaders.findIndex( | |||
| (h) => h.name === "content-disposition" | |||
| ); | |||
| const attachmentHeader = { | |||
| name: "content-disposition", | |||
| value: "attachment", | |||
| }; | |||
| if (foundHeaderIndex) { | |||
| responseHeaders[foundHeaderIndex] = attachmentHeader; | |||
| } else { | |||
| responseHeaders.push(attachmentHeader); | |||
| } | |||
| responseObj = await client.send('Fetch.getResponseBody', { | |||
| requestId, | |||
| }); | |||
| pdfBase64 = Object.values(responseObj); | |||
| this.cndFile = new Buffer(pdfBase64[0], 'base64'); | |||
| if(this.cndFile){ | |||
| this.logger.log('Cnd capturada') | |||
| } | |||
| await client.send('Fetch.continueRequest', { requestId }); | |||
| } else { | |||
| await client.send('Fetch.continueRequest', { requestId }); | |||
| } | |||
| }); | |||
| await this._page.waitFor(25000) | |||
| } | |||
| private async capturandoInformacoesPdf(cndFile: Buffer): Promise<any> { | |||
| let situacao = 0; | |||
| this.logger.log('Extraindo informações do pdf') | |||
| let data = await pdfparse(cndFile) | |||
| if(data.text.includes('CERTIDÃO NEGATIVA')){ | |||
| situacao = 1; | |||
| } else if(data.text.includes('CERTIDÃO POSITIVA COM EFEITOS DE NEGATIVA')) { | |||
| situacao = 4; | |||
| } | |||
| let inicioSeletor = data.text.indexOf('válida até:'); | |||
| const posicaoInicialDataValidade = inicioSeletor + 13; | |||
| const posicaoFinalDataValidade = posicaoInicialDataValidade + 10; | |||
| const dataValidade = new Date(data.text.slice(posicaoInicialDataValidade, posicaoFinalDataValidade)); | |||
| let inicioSeletorDataEmissão = data.text.indexOf('Data da emissão:'); | |||
| const posicaoInicialDataEmissao = inicioSeletorDataEmissão + 18; | |||
| const posicaoFinalDataEmissao = posicaoInicialDataEmissao + 10; | |||
| const dataEmissao = new Date(data.text.slice(posicaoInicialDataEmissao, posicaoFinalDataEmissao)); | |||
| if(dataEmissao && dataValidade && cndFile) { | |||
| this.logger.log('Scraping concluído'); | |||
| this.resultadoScraping = { | |||
| situacao: situacao, | |||
| dataEmissao: getUnixTime(dataEmissao), | |||
| dataValidade: getUnixTime(dataValidade), | |||
| file: cndFile | |||
| } | |||
| } | |||
| } | |||
| private async resolveCaptcha (): Promise<any> { | |||
| const captchaBinary = await this._page.evaluate(() => { | |||
| const img: HTMLImageElement = document.querySelector( | |||
| '[src="/cnd/certidao/geradorcaracteres"]' | |||
| ) | |||
| const canvas = document.createElement('canvas') | |||
| canvas.width = img.width | |||
| canvas.height = img.height | |||
| const ctx = canvas.getContext('2d') | |||
| ctx.drawImage(img, 0, 0) | |||
| return canvas.toDataURL('image/png') | |||
| }) | |||
| const captchaService = new CapMonster() | |||
| const twoCaptchService = new TwoCaptcha() | |||
| const matches = captchaBinary.match(/^data:(.+);base64,(.+)$/) | |||
| if (matches.length !== 3) { | |||
| throw new Error('Erro ao extrair imagem do Captcha') | |||
| } | |||
| let captcha; | |||
| if (await captchaService.temSaldoDisponivel()) { | |||
| this.logger.log(' Usando CapMonster...') | |||
| captcha = await captchaService.resolver(matches[2]); | |||
| } else if (await twoCaptchService.temSaldoDisponivel()) { | |||
| this.logger.log(' Usando TwoCaptcha...') | |||
| captcha = await twoCaptchService.resolveCaptcha(matches[2]); | |||
| } else { | |||
| this.logger.log('Não tem saldo para quebrar o captcha.'); | |||
| return { | |||
| sucesso: false, | |||
| mensagem: 'Não tem saldo para quebrar o captcha.' | |||
| } | |||
| } | |||
| if (!captcha) throw new Error('Erro ao resolver o Captcha'); | |||
| await this._page.type('[name="caracteres"]', captcha, {delay: 250}); | |||
| const selecionandoElementoBotaoOk = await this._page.$$('#spanBotao'); | |||
| await selecionandoElementoBotaoOk[0].click(); | |||
| } | |||
| } | |||
| @@ -0,0 +1 @@ | |||
| declare module '@infosimples/node_two_captcha'; | |||
| @@ -0,0 +1,4 @@ | |||
| { | |||
| "extends": "./tsconfig.json", | |||
| "exclude": ["node_modules", "test", "dist", "**/*spec.ts"] | |||
| } | |||
| @@ -0,0 +1,16 @@ | |||
| { | |||
| "compilerOptions": { | |||
| "module": "commonjs", | |||
| "declaration": true, | |||
| "removeComments": true, | |||
| "emitDecoratorMetadata": true, | |||
| "experimentalDecorators": true, | |||
| "target": "es2017", | |||
| "sourceMap": true, | |||
| "outDir": "./dist", | |||
| "baseUrl": "./", | |||
| "incremental": true, | |||
| "esModuleInterop": true | |||
| }, | |||
| "exclude": ["node_modules", "dist"] | |||
| } | |||