From 084f360ce69c110866550e407375ca07b97ffc87 Mon Sep 17 00:00:00 2001 From: blurpesec Date: Tue, 29 Jan 2019 14:02:22 -0500 Subject: [PATCH 1/2] fixes for github bot --- app.js | 347 ++++++++++++++++++++++++++++++++----- config/config.example.json | 6 +- package.json | 5 +- utils/getSha.js | 24 +++ utils/makeReq.js | 20 +++ utils/server.js | 15 +- utils/webhook.js | 5 +- 7 files changed, 358 insertions(+), 64 deletions(-) create mode 100644 utils/getSha.js create mode 100644 utils/makeReq.js diff --git a/app.js b/app.js index 6c327b3..a4da82f 100644 --- a/app.js +++ b/app.js @@ -1,57 +1,312 @@ const debug = require('debug')('app'); const yaml = require('js-yaml'); +const download = require('download') const config = require('./utils/config'); +const makeReq = require('./utils/makeReq'); const urlScanReport = require('./utils/urlscan'); const app = require('./utils/github'); -const server = require('./utils/server'); const webhook = require('./utils/webhook'); +const getSha = require('./utils/getSha'); -webhook.on('pull_request', async event => { - if (event.payload.action === 'opened') { - debug("New PR opened! (" + event.payload.repository.owner.login + "/" + event.payload.repository.name + "; #" + event.payload.pull_request.number + ")"); - const github = await app.asInstallation(event.payload.installation.id); - debug("Getting original branch..."); - const originalBranch = await github.repos.getContent({ - owner: event.payload.repository.owner.login, - repo: event.payload.repository.name, - ref: event.payload.pull_request.base.ref, - path: '_data/scams.yaml' - }); - debug("Getting PR branch..."); - const pullRequestBranch = await github.repos.getContent({ - owner: event.payload.repository.owner.login, - repo: event.payload.repository.name, - ref: event.payload.pull_request.head.ref, - path: '_data/scams.yaml' - }); - const originalContent = yaml.safeLoad(Buffer.from(originalBranch.data.content,'base64').toString()); - const pullRequestContent = yaml.safeLoad(Buffer.from(pullRequestBranch.data.content,'base64').toString()); - const oldEntries = originalContent.map(entry => entry.url); - const newEntries = await Promise.all(pullRequestContent.map(entry => entry.url).filter(entry => !oldEntries.includes(entry)).map(url => pullRequestContent.find(entry => entry.url === url)).map(async entry => { - entry.URLScan = (await urlScanReport(entry.url)) || '(Error)'; - return entry; - })); - debug("Found " + newEntries.length + " new entries"); - debug("Creating comment..."); - if(newEntries.length > 0) { - await github.issues.createComment({ - owner: event.payload.repository.owner.login, - repo: event.payload.repository.name, - number: event.payload.pull_request.number, - body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") - }); - } else { - await github.issues.createComment({ - owner: event.payload.repository.owner.login, - repo: event.payload.repository.name, - number: event.payload.pull_request.number, - body: '**No new entries added**' - }); - } - debug("Done!"); - } -}); + + + +webhook.on('*', async ({id, name, payload}) => { + if(name === 'pull_request') { + debug('Seeing a new pr now.') + if (payload.action === 'opened') { + debug("New PR opened here") + if (payload.pull_request.user.login === "scamreportbot") { + debug("New PR opened! (" + + "/" + payload.repository.name + "; #" + payload.pull_request.number + ")"); + const github = await app.asInstallation(payload.installation.id); + + /* CommandsFile */ + let originalBranchCommands + let pullRequestBranchCommands; + try { + originalBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/commands/cmd.yaml'); + pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml'); + } catch (e) { + debug("Getting PR branch scams..."); + if(!originalBranchCommands) { + originalBranchCommands = []; + } + if(!pullRequestBranchCommands) { + try { + pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml'); + } catch (e) { + pullRequestBranchCommands = []; + } + } + } + + /* Handle Commands Additions */ + let newCommandsEntries; + if(!originalBranchCommands && !pullRequestBranchCommands) { + newCommandsEntries = null; + } else { + const originalCommandsContent = yaml.safeLoad(Buffer.from(originalBranchCommands,'base64').toString()); + const pullRequestCommandsContent = yaml.safeLoad(Buffer.from(pullRequestBranchCommands,'base64').toString()); + if(originalCommandsContent && pullRequestCommandsContent) { + const oldCommandsEntries = originalCommandsContent.map(entry => entry.data.url); + newCommandsEntries = await Promise.all( + pullRequestCommandsContent.map( + entry => entry.data.url + ).filter( + entry => !oldCommandsEntries.includes(entry) + ).map( + url => pullRequestCommandsContent.find( + entry => entry.data.url === url + ) + ).map(async entry => { + entry.data.URLScan = (await urlScanReport(entry.data.url)) || '(Error)'; + return entry.data; + }) + ); + } else if(!originalCommandsContent && pullRequestCommandsContent){ + if(!pullRequestCommandsContent.length) { + newCommandsEntries = [pullRequestCommandsContent.data]; + } else { + newCommandsEntries = pullRequestCommandsContent.map(entry => { + if (entry.type.toLowerCase() === "ADD".toLowerCase()) { + return entry.data; + } + }) + } + } + + } + + /* Combine ScamsFile and Commands Additions */ + debug("Combining ScamsFile and Commands Additions..."); + const newEntriesArray = []; + await Promise.all( + newCommandsEntries.map(entry => { + newEntriesArray.push(entry); + }) + ); + const newEntriesConst = newEntriesArray; + + /* Download scams file */ + const originalScamsFile = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/data/urls.yaml'); + const parsedOriginalScamsFile = yaml.safeLoad(Buffer.from(originalScamsFile,'base64').toString()); + const newScamsFile = parsedOriginalScamsFile; + await Promise.all( + newEntriesConst.map(entry => { + newScamsFile.push(entry) + }) + ); + + /* Create new commit */ + const newScamsMaterial = await Buffer.from(yaml.safeDump(newScamsFile, { lineWidth: 99999999, indent: 4 })).toString('base64'); + const headUrl = 'https://api.github.com/repos/cryptoscamdb/blacklist/git/trees/' + payload.pull_request.head.sha; + const allTreesPreAdd = JSON.parse(await makeReq(headUrl)); + const dataTreeItemPreAdd = await allTreesPreAdd.tree.find(entry => { + return entry.path === 'data'; + }); + const allDataTrees = JSON.parse(await makeReq(dataTreeItemPreAdd.url)); + const urlsObject = await allDataTrees.tree.find(entry => { + return (entry.path === 'urls.yaml') + }); + + let options = { + owner: 'cryptoscamdb', + repo: 'blacklist', + path: 'data/urls.yaml', + message: 'Added new entry', + content: newScamsMaterial, + sha: urlsObject.sha, + branch: payload.pull_request.head.ref + } + + try { + if (config.autoCommit) { + debug('AutoCommit is still in development - Continuing'); + } else { + debug('AutoCommit is still in development and turned off - Continuing'); + } + } catch (e) { + debug(e); + } + + /* Add URLScan to the message */ + const newEntries = await Promise.all( + newEntriesArray.map(async entry => { + entry.URLScan = (await urlScanReport(entry.url)) || '(Error)'; + return entry; + }) + ); + + const duplicate = false; + newEntries.forEach(entry => { + const duplicateEntry = parsedOriginalScamsFile.find(en => { + return en.url === entry.url + }) + if (duplicateEntry) { + duplicateEntries.push(duplicateEntry) + } + + }); + if (duplicateEntries) { + duplicate = true; + } + + /* Creating comment with new additions */ + if(newEntries.length > 0) { + if (!duplicate) { + await github.issues.createComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + number: payload.pull_request.number, + body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") + }); + } else { + await github.issues.createComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + number: payload.pull_request.number, + body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") + '\n\n' + '**Duplicate entries detected**: \n\n' + duplicateEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") + }); + } + + } else { + await github.issues.createComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + number: payload.pull_request.number, + body: '**No new entries added**' + }); + } + debug("Done!"); + } else { + debug('Entry not made by scamreportbot. Do not auto-commit'); + } + + } + + /* IF PR IS CLOSED AND MERGED */ + if (payload.action === 'closed') { + debug(`Event PR is 'closed'`) + + if (payload.pull_request.merged === true) { + debug('New PR has been merged.'); + debug('Creating update commit'); + const github = await app.asInstallation(payload.installation.id); + + /* CommandsFile */ + let originalBranchCommands + let pullRequestBranchCommands; + try { + originalBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/commands/cmd.yaml'); + } catch (e) { + debug("Getting PR branch scams..."); + originalBranchCommands = []; + pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml'); + } + + /* Handle Commands Additions */ + let newCommandsEntries; + if(!originalBranchCommands && !pullRequestBranchCommands) { + newCommandsEntries = null; + } else { + const originalCommandsContent = yaml.safeLoad(Buffer.from(originalBranchCommands,'base64').toString()); + const pullRequestCommandsContent = yaml.safeLoad(Buffer.from(pullRequestBranchCommands,'base64').toString()); + if(originalCommandsContent && pullRequestCommandsContent) { + const oldCommandsEntries = originalCommandsContent.map(entry => entry.data.url); + newCommandsEntries = await Promise.all( + pullRequestCommandsContent.map( + entry => entry.data.url + ).filter( + entry => !oldCommandsEntries.includes(entry) + ).map( + url => pullRequestCommandsContent.find( + entry => entry.data.url === url + ) + ).map(async entry => { + entry.data.URLScan = (await urlScanReport(entry.data.url)) || '(Error)'; + return entry.data; + }) + ); + debug("Found " + newCommandsEntries.length + " new commands entries"); + debug("Creating comment..."); + } else if(!originalCommandsContent && pullRequestCommandsContent){ + if(!pullRequestCommandsContent.length) { + newCommandsEntries = [pullRequestCommandsContent.data]; + } else { + newCommandsEntries = pullRequestCommandsContent.map(entry => { + return entry.data; + }) + } + } + + } + + /* Combine ScamsFile and Commands Additions */ + debug("Combining ScamsFile and Commands Additions..."); + const newEntriesArray = []; + await Promise.all( + newCommandsEntries.map(entry => { + newEntriesArray.push(entry); + }) + ); + const newEntriesConst = newEntriesArray; + + /* Download scams file */ + const originalScamsFile = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/data/urls.yaml'); + const parsedOriginalScamsFile = yaml.safeLoad(Buffer.from(originalScamsFile,'base64').toString()); + const newScamsFile = parsedOriginalScamsFile; + await Promise.all( + newEntriesConst.map(entry => { + newScamsFile.push(entry) + }) + ); + const newScamsMaterial = await Buffer.from(yaml.safeDump(newScamsFile, { lineWidth: 99999999, indent: 4 })).toString('base64'); + + try { + if (config.autoCommit) { + /* Create new commit */ + debug('Creating update file commit') + let updateOptions = { + owner: 'cryptoscamdb', + repo: 'blacklist', + path: 'data/urls.yaml', + message: 'Added new entry', + content: newScamsMaterial, + sha: await getSha(payload.pull_request.base.sha, 'data', 'urls.yaml'), + branch: payload.pull_request.head.ref + } + await github.repos.updateFile(updateOptions); + } else { + debug('AutoCommit is turned off - Continuing'); + } + } catch (e) { + debug(e); + } + + try { + if (config.deleteCommands) { + /* Creating commands removal commit */ + debug('Creating commands removal commit'); + let commandsOptions = { + owner: 'cryptoscamdb', + repo: 'blacklist', + path: 'commands/cmd.yaml', + message: 'deleted the commands file', + sha: await getSha(payload.pull_request.head.sha, 'commands', 'cmd.yaml'), + branch: 'master' + } + await github.repos.deleteFile(commandsOptions); + } else { + debug('AutoCommit is turned off - Continuing'); + } + } catch (e) { + debug(e); + } + } + } + } +}) process.on('unhandledRejection', reason => { debug(reason); diff --git a/config/config.example.json b/config/config.example.json index e8476db..b0af79e 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -2,5 +2,9 @@ "port": 80, "webhookSecret": "AbcDeFGHiJKLMn12345", "githubAppID": "1", - "urlScanAPIKey": "abcdefgh-01234-5678-ijkl-mnopqrstuvwx" + "urlScanAPIKey": "abcdefgh-01234-5678-ijkl-mnopqrstuvwx", + "githubAccessKey": "0000a000000a0000000a0000000a000000000", + "makeComment": true, + "autoCommit": true, + "deleteCommands": true } \ No newline at end of file diff --git a/package.json b/package.json index 3d55d72..2003bfe 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,15 @@ "url": "git://github.com/CryptoScamDB/github.git" }, "dependencies": { + "@octokit/webhooks": "^5.3.1", "bottleneck": "^2.8.0", "cross-env": "^5.2.0", "debug": "^3.1.0", + "download": "^7.1.0", + "fs": "0.0.1-security", "github-app": "^4.0.1", - "github-webhook-handler": "^0.7.1", "js-yaml": "^3.12.0", + "octonode": "^0.9.5", "request": "^2.88.0" }, "devDependencies": { diff --git a/utils/getSha.js b/utils/getSha.js new file mode 100644 index 0000000..6ad65c1 --- /dev/null +++ b/utils/getSha.js @@ -0,0 +1,24 @@ +const makeReq = require('./makeReq'); +const debug = require('debug')('getSha'); + +const getSha = (headSha, folder, file) => { + return new Promise(async (resolve, reject) => { + try { + const headUrl = 'https://api.github.com/repos/cryptoscamdb/blacklist/git/trees/' + headSha; + const allTreesPreAdd = JSON.parse(await makeReq(headUrl)); + const dataTreeItemPreAdd = await allTreesPreAdd.tree.find(entry => { + return (entry.path === folder); + }); + const allDataTrees = JSON.parse(await makeReq(dataTreeItemPreAdd.url)); + const urlsObject = await allDataTrees.tree.find(entry => { + return (entry.path === file); + }); + resolve(urlsObject.sha); + } catch(err) { + debug(err); + reject(err); + } + }) +} + +module.exports = getSha; \ No newline at end of file diff --git a/utils/makeReq.js b/utils/makeReq.js new file mode 100644 index 0000000..4b91d6b --- /dev/null +++ b/utils/makeReq.js @@ -0,0 +1,20 @@ +const request = require('request'); + +const makeRequest = (url) => { + return new Promise(async (resolve, reject) => { + let options = { + method: "GET", + headers: { + "User-Agent": 'scamreportbot' + } + } + request(url, options, (err, response, body) => { + if(err || response.statusCode === 404 || response.statusCode === 403) reject(err); + else { + resolve(body); + } + }) + }) +} + +module.exports = makeRequest; \ No newline at end of file diff --git a/utils/server.js b/utils/server.js index 1670348..9ceacd8 100644 --- a/utils/server.js +++ b/utils/server.js @@ -1,18 +1,7 @@ const debug = require('debug')('http'); const http = require('http'); const config = require('./config'); -const webhook = require('./webhook'); +const webhooks = require('./webhook'); debug("Registering http server..."); -module.exports = http.createServer((req, res) => { - webhook(req, res, err => { - if (err) { - console.error(err); - res.statusCode = 500; - res.end('500'); - } else { - res.statusCode = 404; - res.end('404'); - } - }); -}).listen(config.port); \ No newline at end of file +module.exports = http.createServer(webhooks.middleware).listen(config.port); \ No newline at end of file diff --git a/utils/webhook.js b/utils/webhook.js index 430aba1..4d7a195 100644 --- a/utils/webhook.js +++ b/utils/webhook.js @@ -1,9 +1,8 @@ const debug = require('debug')('webhook'); -const createWebhook = require('github-webhook-handler'); +const createWebhook = require('@octokit/webhooks'); const config = require('./config'); debug("Registering webhook..."); -module.exports = createWebhook({ - path: '/', +module.exports = new createWebhook({ secret: config.webhookSecret }); \ No newline at end of file From 24f546a9369fbe9fce0fc58d110cfd5ddd058c19 Mon Sep 17 00:00:00 2001 From: blurpesec Date: Wed, 6 Feb 2019 12:23:29 -0500 Subject: [PATCH 2/2] Updated --- app.js | 167 +------------------------------------- utils/createComment.js | 177 +++++++++++++++++++++++++++++++++++++++++ utils/github.js | 20 +++-- utils/webhook.js | 18 ++++- 4 files changed, 210 insertions(+), 172 deletions(-) create mode 100644 utils/createComment.js diff --git a/app.js b/app.js index a4da82f..1f492de 100644 --- a/app.js +++ b/app.js @@ -1,14 +1,14 @@ const debug = require('debug')('app'); const yaml = require('js-yaml'); const download = require('download') - +const server = require('./utils/server'); const config = require('./utils/config'); const makeReq = require('./utils/makeReq'); const urlScanReport = require('./utils/urlscan'); const app = require('./utils/github'); const webhook = require('./utils/webhook'); const getSha = require('./utils/getSha'); - +const createComment = require('./utils/createComment'); @@ -18,171 +18,12 @@ webhook.on('*', async ({id, name, payload}) => { if (payload.action === 'opened') { debug("New PR opened here") if (payload.pull_request.user.login === "scamreportbot") { - debug("New PR opened! (" + + "/" + payload.repository.name + "; #" + payload.pull_request.number + ")"); - const github = await app.asInstallation(payload.installation.id); - - /* CommandsFile */ - let originalBranchCommands - let pullRequestBranchCommands; - try { - originalBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/commands/cmd.yaml'); - pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml'); - } catch (e) { - debug("Getting PR branch scams..."); - if(!originalBranchCommands) { - originalBranchCommands = []; - } - if(!pullRequestBranchCommands) { - try { - pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml'); - } catch (e) { - pullRequestBranchCommands = []; - } - } - } - - /* Handle Commands Additions */ - let newCommandsEntries; - if(!originalBranchCommands && !pullRequestBranchCommands) { - newCommandsEntries = null; - } else { - const originalCommandsContent = yaml.safeLoad(Buffer.from(originalBranchCommands,'base64').toString()); - const pullRequestCommandsContent = yaml.safeLoad(Buffer.from(pullRequestBranchCommands,'base64').toString()); - if(originalCommandsContent && pullRequestCommandsContent) { - const oldCommandsEntries = originalCommandsContent.map(entry => entry.data.url); - newCommandsEntries = await Promise.all( - pullRequestCommandsContent.map( - entry => entry.data.url - ).filter( - entry => !oldCommandsEntries.includes(entry) - ).map( - url => pullRequestCommandsContent.find( - entry => entry.data.url === url - ) - ).map(async entry => { - entry.data.URLScan = (await urlScanReport(entry.data.url)) || '(Error)'; - return entry.data; - }) - ); - } else if(!originalCommandsContent && pullRequestCommandsContent){ - if(!pullRequestCommandsContent.length) { - newCommandsEntries = [pullRequestCommandsContent.data]; - } else { - newCommandsEntries = pullRequestCommandsContent.map(entry => { - if (entry.type.toLowerCase() === "ADD".toLowerCase()) { - return entry.data; - } - }) - } - } - - } - - /* Combine ScamsFile and Commands Additions */ - debug("Combining ScamsFile and Commands Additions..."); - const newEntriesArray = []; - await Promise.all( - newCommandsEntries.map(entry => { - newEntriesArray.push(entry); - }) - ); - const newEntriesConst = newEntriesArray; - - /* Download scams file */ - const originalScamsFile = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/data/urls.yaml'); - const parsedOriginalScamsFile = yaml.safeLoad(Buffer.from(originalScamsFile,'base64').toString()); - const newScamsFile = parsedOriginalScamsFile; - await Promise.all( - newEntriesConst.map(entry => { - newScamsFile.push(entry) - }) - ); - - /* Create new commit */ - const newScamsMaterial = await Buffer.from(yaml.safeDump(newScamsFile, { lineWidth: 99999999, indent: 4 })).toString('base64'); - const headUrl = 'https://api.github.com/repos/cryptoscamdb/blacklist/git/trees/' + payload.pull_request.head.sha; - const allTreesPreAdd = JSON.parse(await makeReq(headUrl)); - const dataTreeItemPreAdd = await allTreesPreAdd.tree.find(entry => { - return entry.path === 'data'; - }); - const allDataTrees = JSON.parse(await makeReq(dataTreeItemPreAdd.url)); - const urlsObject = await allDataTrees.tree.find(entry => { - return (entry.path === 'urls.yaml') - }); - - let options = { - owner: 'cryptoscamdb', - repo: 'blacklist', - path: 'data/urls.yaml', - message: 'Added new entry', - content: newScamsMaterial, - sha: urlsObject.sha, - branch: payload.pull_request.head.ref - } - - try { - if (config.autoCommit) { - debug('AutoCommit is still in development - Continuing'); - } else { - debug('AutoCommit is still in development and turned off - Continuing'); - } - } catch (e) { - debug(e); - } - - /* Add URLScan to the message */ - const newEntries = await Promise.all( - newEntriesArray.map(async entry => { - entry.URLScan = (await urlScanReport(entry.url)) || '(Error)'; - return entry; - }) - ); - - const duplicate = false; - newEntries.forEach(entry => { - const duplicateEntry = parsedOriginalScamsFile.find(en => { - return en.url === entry.url - }) - if (duplicateEntry) { - duplicateEntries.push(duplicateEntry) - } - - }); - if (duplicateEntries) { - duplicate = true; - } - - /* Creating comment with new additions */ - if(newEntries.length > 0) { - if (!duplicate) { - await github.issues.createComment({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - number: payload.pull_request.number, - body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") - }); - } else { - await github.issues.createComment({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - number: payload.pull_request.number, - body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") + '\n\n' + '**Duplicate entries detected**: \n\n' + duplicateEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") - }); - } - - } else { - await github.issues.createComment({ - owner: payload.repository.owner.login, - repo: payload.repository.name, - number: payload.pull_request.number, - body: '**No new entries added**' - }); - } - debug("Done!"); + createComment(payload); } else { debug('Entry not made by scamreportbot. Do not auto-commit'); } + } /* IF PR IS CLOSED AND MERGED */ diff --git a/utils/createComment.js b/utils/createComment.js new file mode 100644 index 0000000..d77a655 --- /dev/null +++ b/utils/createComment.js @@ -0,0 +1,177 @@ +const debug = require('debug')('createContent'); +const yaml = require('js-yaml'); +const download = require('download') +const urlScanReport = require('./urlscan'); +const app = require('./github'); +const server = require('./server'); +const config = require('./config'); +const makeReq = require('./makeReq'); + + +const createComment = (payload) => { + return new Promise(async (resolve, reject) => { + debug("New PR opened! (" + + "/" + payload.repository.name + "; #" + payload.pull_request.number + ")"); + const github = await app.asInstallation(payload.installation.id); + + /* CommandsFile */ + let originalBranchCommands + let pullRequestBranchCommands; + try { + originalBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/commands/cmd.yaml'); + pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml'); + } catch (e) { + debug("Getting PR branch scams..."); + if(!originalBranchCommands) { + originalBranchCommands = []; + } + if(!pullRequestBranchCommands) { + try { + pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml'); + } catch (e) { + pullRequestBranchCommands = []; + } + } + } + + /* Handle Commands Additions */ + let newCommandsEntries; + if(!originalBranchCommands && !pullRequestBranchCommands) { + newCommandsEntries = null; + } else { + const originalCommandsContent = yaml.safeLoad(Buffer.from(originalBranchCommands,'base64').toString()); + const pullRequestCommandsContent = yaml.safeLoad(Buffer.from(pullRequestBranchCommands,'base64').toString()); + if(originalCommandsContent && pullRequestCommandsContent) { + const oldCommandsEntries = originalCommandsContent.map(entry => entry.data.url); + newCommandsEntries = await Promise.all( + pullRequestCommandsContent.map( + entry => entry.data.url + ).filter( + entry => !oldCommandsEntries.includes(entry) + ).map( + url => pullRequestCommandsContent.find( + entry => entry.data.url === url + ) + ).map(async entry => { + entry.data.URLScan = (await urlScanReport(entry.data.url)) || '(Error)'; + return entry.data; + }) + ); + } else if(!originalCommandsContent && pullRequestCommandsContent){ + if(!pullRequestCommandsContent.length) { + newCommandsEntries = [pullRequestCommandsContent.data]; + } else { + newCommandsEntries = pullRequestCommandsContent.map(entry => { + if (entry.type.toLowerCase() === "ADD".toLowerCase()) { + return entry.data; + } + }) + } + } + + } + + /* Combine ScamsFile and Commands Additions */ + debug("Combining ScamsFile and Commands Additions..."); + const newEntriesArray = []; + await Promise.all( + newCommandsEntries.map(entry => { + newEntriesArray.push(entry); + }) + ); + const newEntriesConst = newEntriesArray; + + /* Download scams file */ + const originalScamsFile = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/data/urls.yaml'); + const parsedOriginalScamsFile = await yaml.safeLoad(Buffer.from(originalScamsFile,'base64').toString()); + const newScamsFile = await JSON.parse(JSON.stringify(parsedOriginalScamsFile)); + await Promise.all( + newEntriesConst.map(entry => { + newScamsFile.push(entry) + }) + ); + + /* Create new commit */ + const newScamsMaterial = await Buffer.from(yaml.safeDump(newScamsFile, { lineWidth: 99999999, indent: 4 })).toString('base64'); + const headUrl = 'https://api.github.com/repos/cryptoscamdb/blacklist/git/trees/' + payload.pull_request.head.sha; + const allTreesPreAdd = JSON.parse(await makeReq(headUrl)); + const dataTreeItemPreAdd = await allTreesPreAdd.tree.find(entry => { + return entry.path === 'data'; + }); + const allDataTrees = JSON.parse(await makeReq(dataTreeItemPreAdd.url)); + const urlsObject = await allDataTrees.tree.find(entry => { + return (entry.path === 'urls.yaml') + }); + + let options = { + owner: 'cryptoscamdb', + repo: 'blacklist', + path: 'data/urls.yaml', + message: 'Added new entry', + content: newScamsMaterial, + sha: urlsObject.sha, + branch: payload.pull_request.head.ref + } + + try { + if (config.autoCommit) { + debug('AutoCommit is still in development - Continuing'); + } else { + debug('AutoCommit is still in development and turned off - Continuing'); + } + } catch (e) { + debug(e); + } + + /* Add URLScan to the message */ + const newEntries = await Promise.all( + newEntriesArray.map(async entry => { + entry.URLScan = (await urlScanReport(entry.url)) || '(Error)'; + return entry; + }) + ); + let duplicate = false; + const duplicateEntries = []; + await Promise.all( + await newEntries.map(async entry => { + parsedOriginalScamsFile.map(en => { + if (en.url.toLowerCase() === entry.url.toLowerCase()) { + duplicateEntries.push(entry); + }; + }); + }) + ); + + duplicate = duplicateEntries.length >= 1 ? true : false; + + /* Creating comment with new additions */ + if(newEntries.length > 0) { + debug('Making Comments now') + if (!duplicate) { + await github.issues.createComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + number: payload.pull_request.number, + body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") + }); + } else { + await github.issues.createComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + number: payload.pull_request.number, + body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") + '\n\n' + '**Duplicate entries detected**: \n\n' + duplicateEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") + }); + } + + } else { + await github.issues.createComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + number: payload.pull_request.number, + body: '**No new entries added**' + }); + } + debug("Done!"); + }) +} + +module.exports = createComment; \ No newline at end of file diff --git a/utils/github.js b/utils/github.js index facc25f..b8638c2 100644 --- a/utils/github.js +++ b/utils/github.js @@ -3,8 +3,18 @@ const createGitHubApp = require('github-app'); const config = require('./config'); const privateKey = require('./privatekey'); -debug("Registering Github app..."); -module.exports = createGitHubApp({ - id: config.githubAppID, - cert: privateKey -}); \ No newline at end of file + +const createGHApp = () => { + try { + debug("Registering Github app..."); + return (createGitHubApp({ + id: config.githubAppID, + cert: privateKey + })); + debug("Finished registering Github app."); + } catch (err) { + debug(err); + } +} + +module.exports = createGHApp(); \ No newline at end of file diff --git a/utils/webhook.js b/utils/webhook.js index 4d7a195..fac1748 100644 --- a/utils/webhook.js +++ b/utils/webhook.js @@ -2,7 +2,17 @@ const debug = require('debug')('webhook'); const createWebhook = require('@octokit/webhooks'); const config = require('./config'); -debug("Registering webhook..."); -module.exports = new createWebhook({ - secret: config.webhookSecret -}); \ No newline at end of file +const createGHWebhook = () => { + try { + debug("Registering webhook..."); + return (new createWebhook({ + secret: config.webhookSecret + })); + debug("Webhook registered") + } catch (err) { + debug(err); + } +} + + +module.exports = createGHWebhook(); \ No newline at end of file