diff --git a/app.js b/app.js index 6c327b3..1f492de 100644 --- a/app.js +++ b/app.js @@ -1,57 +1,153 @@ const debug = require('debug')('app'); const yaml = require('js-yaml'); - +const download = require('download') +const server = require('./utils/server'); const config = require('./utils/config'); +const makeReq = require('./utils/makeReq'); const urlScanReport = require('./utils/urlscan'); const app = require('./utils/github'); -const server = require('./utils/server'); const webhook = require('./utils/webhook'); +const getSha = require('./utils/getSha'); +const createComment = require('./utils/createComment'); -webhook.on('pull_request', async event => { - if (event.payload.action === 'opened') { - debug("New PR opened! (" + event.payload.repository.owner.login + "/" + event.payload.repository.name + "; #" + event.payload.pull_request.number + ")"); - const github = await app.asInstallation(event.payload.installation.id); - debug("Getting original branch..."); - const originalBranch = await github.repos.getContent({ - owner: event.payload.repository.owner.login, - repo: event.payload.repository.name, - ref: event.payload.pull_request.base.ref, - path: '_data/scams.yaml' - }); - debug("Getting PR branch..."); - const pullRequestBranch = await github.repos.getContent({ - owner: event.payload.repository.owner.login, - repo: event.payload.repository.name, - ref: event.payload.pull_request.head.ref, - path: '_data/scams.yaml' - }); - const originalContent = yaml.safeLoad(Buffer.from(originalBranch.data.content,'base64').toString()); - const pullRequestContent = yaml.safeLoad(Buffer.from(pullRequestBranch.data.content,'base64').toString()); - const oldEntries = originalContent.map(entry => entry.url); - const newEntries = await Promise.all(pullRequestContent.map(entry => entry.url).filter(entry => !oldEntries.includes(entry)).map(url => pullRequestContent.find(entry => entry.url === url)).map(async entry => { - entry.URLScan = (await urlScanReport(entry.url)) || '(Error)'; - return entry; - })); - debug("Found " + newEntries.length + " new entries"); - debug("Creating comment..."); - if(newEntries.length > 0) { - await github.issues.createComment({ - owner: event.payload.repository.owner.login, - repo: event.payload.repository.name, - number: event.payload.pull_request.number, - body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") - }); - } else { - await github.issues.createComment({ - owner: event.payload.repository.owner.login, - repo: event.payload.repository.name, - number: event.payload.pull_request.number, - body: '**No new entries added**' - }); - } - debug("Done!"); - } -}); + + +webhook.on('*', async ({id, name, payload}) => { + if(name === 'pull_request') { + debug('Seeing a new pr now.') + if (payload.action === 'opened') { + debug("New PR opened here") + if (payload.pull_request.user.login === "scamreportbot") { + createComment(payload); + } else { + debug('Entry not made by scamreportbot. Do not auto-commit'); + } + + + } + + /* IF PR IS CLOSED AND MERGED */ + if (payload.action === 'closed') { + debug(`Event PR is 'closed'`) + + if (payload.pull_request.merged === true) { + debug('New PR has been merged.'); + debug('Creating update commit'); + const github = await app.asInstallation(payload.installation.id); + + /* CommandsFile */ + let originalBranchCommands + let pullRequestBranchCommands; + try { + originalBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/commands/cmd.yaml'); + } catch (e) { + debug("Getting PR branch scams..."); + originalBranchCommands = []; + pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml'); + } + + /* Handle Commands Additions */ + let newCommandsEntries; + if(!originalBranchCommands && !pullRequestBranchCommands) { + newCommandsEntries = null; + } else { + const originalCommandsContent = yaml.safeLoad(Buffer.from(originalBranchCommands,'base64').toString()); + const pullRequestCommandsContent = yaml.safeLoad(Buffer.from(pullRequestBranchCommands,'base64').toString()); + if(originalCommandsContent && pullRequestCommandsContent) { + const oldCommandsEntries = originalCommandsContent.map(entry => entry.data.url); + newCommandsEntries = await Promise.all( + pullRequestCommandsContent.map( + entry => entry.data.url + ).filter( + entry => !oldCommandsEntries.includes(entry) + ).map( + url => pullRequestCommandsContent.find( + entry => entry.data.url === url + ) + ).map(async entry => { + entry.data.URLScan = (await urlScanReport(entry.data.url)) || '(Error)'; + return entry.data; + }) + ); + debug("Found " + newCommandsEntries.length + " new commands entries"); + debug("Creating comment..."); + } else if(!originalCommandsContent && pullRequestCommandsContent){ + if(!pullRequestCommandsContent.length) { + newCommandsEntries = [pullRequestCommandsContent.data]; + } else { + newCommandsEntries = pullRequestCommandsContent.map(entry => { + return entry.data; + }) + } + } + + } + + /* Combine ScamsFile and Commands Additions */ + debug("Combining ScamsFile and Commands Additions..."); + const newEntriesArray = []; + await Promise.all( + newCommandsEntries.map(entry => { + newEntriesArray.push(entry); + }) + ); + const newEntriesConst = newEntriesArray; + + /* Download scams file */ + const originalScamsFile = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/data/urls.yaml'); + const parsedOriginalScamsFile = yaml.safeLoad(Buffer.from(originalScamsFile,'base64').toString()); + const newScamsFile = parsedOriginalScamsFile; + await Promise.all( + newEntriesConst.map(entry => { + newScamsFile.push(entry) + }) + ); + const newScamsMaterial = await Buffer.from(yaml.safeDump(newScamsFile, { lineWidth: 99999999, indent: 4 })).toString('base64'); + + try { + if (config.autoCommit) { + /* Create new commit */ + debug('Creating update file commit') + let updateOptions = { + owner: 'cryptoscamdb', + repo: 'blacklist', + path: 'data/urls.yaml', + message: 'Added new entry', + content: newScamsMaterial, + sha: await getSha(payload.pull_request.base.sha, 'data', 'urls.yaml'), + branch: payload.pull_request.head.ref + } + await github.repos.updateFile(updateOptions); + } else { + debug('AutoCommit is turned off - Continuing'); + } + } catch (e) { + debug(e); + } + + try { + if (config.deleteCommands) { + /* Creating commands removal commit */ + debug('Creating commands removal commit'); + let commandsOptions = { + owner: 'cryptoscamdb', + repo: 'blacklist', + path: 'commands/cmd.yaml', + message: 'deleted the commands file', + sha: await getSha(payload.pull_request.head.sha, 'commands', 'cmd.yaml'), + branch: 'master' + } + await github.repos.deleteFile(commandsOptions); + } else { + debug('AutoCommit is turned off - Continuing'); + } + } catch (e) { + debug(e); + } + } + } + } +}) process.on('unhandledRejection', reason => { debug(reason); diff --git a/config/config.example.json b/config/config.example.json index e8476db..b0af79e 100644 --- a/config/config.example.json +++ b/config/config.example.json @@ -2,5 +2,9 @@ "port": 80, "webhookSecret": "AbcDeFGHiJKLMn12345", "githubAppID": "1", - "urlScanAPIKey": "abcdefgh-01234-5678-ijkl-mnopqrstuvwx" + "urlScanAPIKey": "abcdefgh-01234-5678-ijkl-mnopqrstuvwx", + "githubAccessKey": "0000a000000a0000000a0000000a000000000", + "makeComment": true, + "autoCommit": true, + "deleteCommands": true } \ No newline at end of file diff --git a/package.json b/package.json index 3d55d72..2003bfe 100644 --- a/package.json +++ b/package.json @@ -17,12 +17,15 @@ "url": "git://github.com/CryptoScamDB/github.git" }, "dependencies": { + "@octokit/webhooks": "^5.3.1", "bottleneck": "^2.8.0", "cross-env": "^5.2.0", "debug": "^3.1.0", + "download": "^7.1.0", + "fs": "0.0.1-security", "github-app": "^4.0.1", - "github-webhook-handler": "^0.7.1", "js-yaml": "^3.12.0", + "octonode": "^0.9.5", "request": "^2.88.0" }, "devDependencies": { diff --git a/utils/createComment.js b/utils/createComment.js new file mode 100644 index 0000000..d77a655 --- /dev/null +++ b/utils/createComment.js @@ -0,0 +1,177 @@ +const debug = require('debug')('createContent'); +const yaml = require('js-yaml'); +const download = require('download') +const urlScanReport = require('./urlscan'); +const app = require('./github'); +const server = require('./server'); +const config = require('./config'); +const makeReq = require('./makeReq'); + + +const createComment = (payload) => { + return new Promise(async (resolve, reject) => { + debug("New PR opened! (" + + "/" + payload.repository.name + "; #" + payload.pull_request.number + ")"); + const github = await app.asInstallation(payload.installation.id); + + /* CommandsFile */ + let originalBranchCommands + let pullRequestBranchCommands; + try { + originalBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/commands/cmd.yaml'); + pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml'); + } catch (e) { + debug("Getting PR branch scams..."); + if(!originalBranchCommands) { + originalBranchCommands = []; + } + if(!pullRequestBranchCommands) { + try { + pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml'); + } catch (e) { + pullRequestBranchCommands = []; + } + } + } + + /* Handle Commands Additions */ + let newCommandsEntries; + if(!originalBranchCommands && !pullRequestBranchCommands) { + newCommandsEntries = null; + } else { + const originalCommandsContent = yaml.safeLoad(Buffer.from(originalBranchCommands,'base64').toString()); + const pullRequestCommandsContent = yaml.safeLoad(Buffer.from(pullRequestBranchCommands,'base64').toString()); + if(originalCommandsContent && pullRequestCommandsContent) { + const oldCommandsEntries = originalCommandsContent.map(entry => entry.data.url); + newCommandsEntries = await Promise.all( + pullRequestCommandsContent.map( + entry => entry.data.url + ).filter( + entry => !oldCommandsEntries.includes(entry) + ).map( + url => pullRequestCommandsContent.find( + entry => entry.data.url === url + ) + ).map(async entry => { + entry.data.URLScan = (await urlScanReport(entry.data.url)) || '(Error)'; + return entry.data; + }) + ); + } else if(!originalCommandsContent && pullRequestCommandsContent){ + if(!pullRequestCommandsContent.length) { + newCommandsEntries = [pullRequestCommandsContent.data]; + } else { + newCommandsEntries = pullRequestCommandsContent.map(entry => { + if (entry.type.toLowerCase() === "ADD".toLowerCase()) { + return entry.data; + } + }) + } + } + + } + + /* Combine ScamsFile and Commands Additions */ + debug("Combining ScamsFile and Commands Additions..."); + const newEntriesArray = []; + await Promise.all( + newCommandsEntries.map(entry => { + newEntriesArray.push(entry); + }) + ); + const newEntriesConst = newEntriesArray; + + /* Download scams file */ + const originalScamsFile = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/data/urls.yaml'); + const parsedOriginalScamsFile = await yaml.safeLoad(Buffer.from(originalScamsFile,'base64').toString()); + const newScamsFile = await JSON.parse(JSON.stringify(parsedOriginalScamsFile)); + await Promise.all( + newEntriesConst.map(entry => { + newScamsFile.push(entry) + }) + ); + + /* Create new commit */ + const newScamsMaterial = await Buffer.from(yaml.safeDump(newScamsFile, { lineWidth: 99999999, indent: 4 })).toString('base64'); + const headUrl = 'https://api.github.com/repos/cryptoscamdb/blacklist/git/trees/' + payload.pull_request.head.sha; + const allTreesPreAdd = JSON.parse(await makeReq(headUrl)); + const dataTreeItemPreAdd = await allTreesPreAdd.tree.find(entry => { + return entry.path === 'data'; + }); + const allDataTrees = JSON.parse(await makeReq(dataTreeItemPreAdd.url)); + const urlsObject = await allDataTrees.tree.find(entry => { + return (entry.path === 'urls.yaml') + }); + + let options = { + owner: 'cryptoscamdb', + repo: 'blacklist', + path: 'data/urls.yaml', + message: 'Added new entry', + content: newScamsMaterial, + sha: urlsObject.sha, + branch: payload.pull_request.head.ref + } + + try { + if (config.autoCommit) { + debug('AutoCommit is still in development - Continuing'); + } else { + debug('AutoCommit is still in development and turned off - Continuing'); + } + } catch (e) { + debug(e); + } + + /* Add URLScan to the message */ + const newEntries = await Promise.all( + newEntriesArray.map(async entry => { + entry.URLScan = (await urlScanReport(entry.url)) || '(Error)'; + return entry; + }) + ); + let duplicate = false; + const duplicateEntries = []; + await Promise.all( + await newEntries.map(async entry => { + parsedOriginalScamsFile.map(en => { + if (en.url.toLowerCase() === entry.url.toLowerCase()) { + duplicateEntries.push(entry); + }; + }); + }) + ); + + duplicate = duplicateEntries.length >= 1 ? true : false; + + /* Creating comment with new additions */ + if(newEntries.length > 0) { + debug('Making Comments now') + if (!duplicate) { + await github.issues.createComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + number: payload.pull_request.number, + body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") + }); + } else { + await github.issues.createComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + number: payload.pull_request.number, + body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") + '\n\n' + '**Duplicate entries detected**: \n\n' + duplicateEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") + }); + } + + } else { + await github.issues.createComment({ + owner: payload.repository.owner.login, + repo: payload.repository.name, + number: payload.pull_request.number, + body: '**No new entries added**' + }); + } + debug("Done!"); + }) +} + +module.exports = createComment; \ No newline at end of file diff --git a/utils/getSha.js b/utils/getSha.js new file mode 100644 index 0000000..6ad65c1 --- /dev/null +++ b/utils/getSha.js @@ -0,0 +1,24 @@ +const makeReq = require('./makeReq'); +const debug = require('debug')('getSha'); + +const getSha = (headSha, folder, file) => { + return new Promise(async (resolve, reject) => { + try { + const headUrl = 'https://api.github.com/repos/cryptoscamdb/blacklist/git/trees/' + headSha; + const allTreesPreAdd = JSON.parse(await makeReq(headUrl)); + const dataTreeItemPreAdd = await allTreesPreAdd.tree.find(entry => { + return (entry.path === folder); + }); + const allDataTrees = JSON.parse(await makeReq(dataTreeItemPreAdd.url)); + const urlsObject = await allDataTrees.tree.find(entry => { + return (entry.path === file); + }); + resolve(urlsObject.sha); + } catch(err) { + debug(err); + reject(err); + } + }) +} + +module.exports = getSha; \ No newline at end of file diff --git a/utils/github.js b/utils/github.js index facc25f..b8638c2 100644 --- a/utils/github.js +++ b/utils/github.js @@ -3,8 +3,18 @@ const createGitHubApp = require('github-app'); const config = require('./config'); const privateKey = require('./privatekey'); -debug("Registering Github app..."); -module.exports = createGitHubApp({ - id: config.githubAppID, - cert: privateKey -}); \ No newline at end of file + +const createGHApp = () => { + try { + debug("Registering Github app..."); + return (createGitHubApp({ + id: config.githubAppID, + cert: privateKey + })); + debug("Finished registering Github app."); + } catch (err) { + debug(err); + } +} + +module.exports = createGHApp(); \ No newline at end of file diff --git a/utils/makeReq.js b/utils/makeReq.js new file mode 100644 index 0000000..4b91d6b --- /dev/null +++ b/utils/makeReq.js @@ -0,0 +1,20 @@ +const request = require('request'); + +const makeRequest = (url) => { + return new Promise(async (resolve, reject) => { + let options = { + method: "GET", + headers: { + "User-Agent": 'scamreportbot' + } + } + request(url, options, (err, response, body) => { + if(err || response.statusCode === 404 || response.statusCode === 403) reject(err); + else { + resolve(body); + } + }) + }) +} + +module.exports = makeRequest; \ No newline at end of file diff --git a/utils/server.js b/utils/server.js index 1670348..9ceacd8 100644 --- a/utils/server.js +++ b/utils/server.js @@ -1,18 +1,7 @@ const debug = require('debug')('http'); const http = require('http'); const config = require('./config'); -const webhook = require('./webhook'); +const webhooks = require('./webhook'); debug("Registering http server..."); -module.exports = http.createServer((req, res) => { - webhook(req, res, err => { - if (err) { - console.error(err); - res.statusCode = 500; - res.end('500'); - } else { - res.statusCode = 404; - res.end('404'); - } - }); -}).listen(config.port); \ No newline at end of file +module.exports = http.createServer(webhooks.middleware).listen(config.port); \ No newline at end of file diff --git a/utils/webhook.js b/utils/webhook.js index 430aba1..fac1748 100644 --- a/utils/webhook.js +++ b/utils/webhook.js @@ -1,9 +1,18 @@ const debug = require('debug')('webhook'); -const createWebhook = require('github-webhook-handler'); +const createWebhook = require('@octokit/webhooks'); const config = require('./config'); -debug("Registering webhook..."); -module.exports = createWebhook({ - path: '/', - secret: config.webhookSecret -}); \ No newline at end of file +const createGHWebhook = () => { + try { + debug("Registering webhook..."); + return (new createWebhook({ + secret: config.webhookSecret + })); + debug("Webhook registered") + } catch (err) { + debug(err); + } +} + + +module.exports = createGHWebhook(); \ No newline at end of file