diff --git a/app.js b/app.js
index 6c327b3..1f492de 100644
--- a/app.js
+++ b/app.js
@@ -1,57 +1,153 @@
const debug = require('debug')('app');
const yaml = require('js-yaml');
-
+const download = require('download')
+const server = require('./utils/server');
const config = require('./utils/config');
+const makeReq = require('./utils/makeReq');
const urlScanReport = require('./utils/urlscan');
const app = require('./utils/github');
-const server = require('./utils/server');
const webhook = require('./utils/webhook');
+const getSha = require('./utils/getSha');
+const createComment = require('./utils/createComment');
-webhook.on('pull_request', async event => {
- if (event.payload.action === 'opened') {
- debug("New PR opened! (" + event.payload.repository.owner.login + "/" + event.payload.repository.name + "; #" + event.payload.pull_request.number + ")");
- const github = await app.asInstallation(event.payload.installation.id);
- debug("Getting original branch...");
- const originalBranch = await github.repos.getContent({
- owner: event.payload.repository.owner.login,
- repo: event.payload.repository.name,
- ref: event.payload.pull_request.base.ref,
- path: '_data/scams.yaml'
- });
- debug("Getting PR branch...");
- const pullRequestBranch = await github.repos.getContent({
- owner: event.payload.repository.owner.login,
- repo: event.payload.repository.name,
- ref: event.payload.pull_request.head.ref,
- path: '_data/scams.yaml'
- });
- const originalContent = yaml.safeLoad(Buffer.from(originalBranch.data.content,'base64').toString());
- const pullRequestContent = yaml.safeLoad(Buffer.from(pullRequestBranch.data.content,'base64').toString());
- const oldEntries = originalContent.map(entry => entry.url);
- const newEntries = await Promise.all(pullRequestContent.map(entry => entry.url).filter(entry => !oldEntries.includes(entry)).map(url => pullRequestContent.find(entry => entry.url === url)).map(async entry => {
- entry.URLScan = (await urlScanReport(entry.url)) || '(Error)';
- return entry;
- }));
- debug("Found " + newEntries.length + " new entries");
- debug("Creating comment...");
- if(newEntries.length > 0) {
- await github.issues.createComment({
- owner: event.payload.repository.owner.login,
- repo: event.payload.repository.name,
- number: event.payload.pull_request.number,
- body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n")
- });
- } else {
- await github.issues.createComment({
- owner: event.payload.repository.owner.login,
- repo: event.payload.repository.name,
- number: event.payload.pull_request.number,
- body: '**No new entries added**'
- });
- }
- debug("Done!");
- }
-});
+
+
+webhook.on('*', async ({id, name, payload}) => {
+ if(name === 'pull_request') {
+ debug('Seeing a new pr now.')
+ if (payload.action === 'opened') {
+ debug("New PR opened here")
+ if (payload.pull_request.user.login === "scamreportbot") {
+ createComment(payload);
+ } else {
+ debug('Entry not made by scamreportbot. Do not auto-commit');
+ }
+
+
+ }
+
+ /* IF PR IS CLOSED AND MERGED */
+ if (payload.action === 'closed') {
+ debug(`Event PR is 'closed'`)
+
+ if (payload.pull_request.merged === true) {
+ debug('New PR has been merged.');
+ debug('Creating update commit');
+ const github = await app.asInstallation(payload.installation.id);
+
+ /* CommandsFile */
+ let originalBranchCommands
+ let pullRequestBranchCommands;
+ try {
+ originalBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/commands/cmd.yaml');
+ } catch (e) {
+ debug("Getting PR branch scams...");
+ originalBranchCommands = [];
+ pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml');
+ }
+
+ /* Handle Commands Additions */
+ let newCommandsEntries;
+ if(!originalBranchCommands && !pullRequestBranchCommands) {
+ newCommandsEntries = null;
+ } else {
+ const originalCommandsContent = yaml.safeLoad(Buffer.from(originalBranchCommands,'base64').toString());
+ const pullRequestCommandsContent = yaml.safeLoad(Buffer.from(pullRequestBranchCommands,'base64').toString());
+ if(originalCommandsContent && pullRequestCommandsContent) {
+ const oldCommandsEntries = originalCommandsContent.map(entry => entry.data.url);
+ newCommandsEntries = await Promise.all(
+ pullRequestCommandsContent.map(
+ entry => entry.data.url
+ ).filter(
+ entry => !oldCommandsEntries.includes(entry)
+ ).map(
+ url => pullRequestCommandsContent.find(
+ entry => entry.data.url === url
+ )
+ ).map(async entry => {
+ entry.data.URLScan = (await urlScanReport(entry.data.url)) || '(Error)';
+ return entry.data;
+ })
+ );
+ debug("Found " + newCommandsEntries.length + " new commands entries");
+ debug("Creating comment...");
+ } else if(!originalCommandsContent && pullRequestCommandsContent){
+ if(!pullRequestCommandsContent.length) {
+ newCommandsEntries = [pullRequestCommandsContent.data];
+ } else {
+ newCommandsEntries = pullRequestCommandsContent.map(entry => {
+ return entry.data;
+ })
+ }
+ }
+
+ }
+
+ /* Combine ScamsFile and Commands Additions */
+ debug("Combining ScamsFile and Commands Additions...");
+ const newEntriesArray = [];
+ await Promise.all(
+ newCommandsEntries.map(entry => {
+ newEntriesArray.push(entry);
+ })
+ );
+ const newEntriesConst = newEntriesArray;
+
+ /* Download scams file */
+ const originalScamsFile = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/data/urls.yaml');
+ const parsedOriginalScamsFile = yaml.safeLoad(Buffer.from(originalScamsFile,'base64').toString());
+ const newScamsFile = parsedOriginalScamsFile;
+ await Promise.all(
+ newEntriesConst.map(entry => {
+ newScamsFile.push(entry)
+ })
+ );
+ const newScamsMaterial = await Buffer.from(yaml.safeDump(newScamsFile, { lineWidth: 99999999, indent: 4 })).toString('base64');
+
+ try {
+ if (config.autoCommit) {
+ /* Create new commit */
+ debug('Creating update file commit')
+ let updateOptions = {
+ owner: 'cryptoscamdb',
+ repo: 'blacklist',
+ path: 'data/urls.yaml',
+ message: 'Added new entry',
+ content: newScamsMaterial,
+ sha: await getSha(payload.pull_request.base.sha, 'data', 'urls.yaml'),
+ branch: payload.pull_request.head.ref
+ }
+ await github.repos.updateFile(updateOptions);
+ } else {
+ debug('AutoCommit is turned off - Continuing');
+ }
+ } catch (e) {
+ debug(e);
+ }
+
+ try {
+ if (config.deleteCommands) {
+ /* Creating commands removal commit */
+ debug('Creating commands removal commit');
+ let commandsOptions = {
+ owner: 'cryptoscamdb',
+ repo: 'blacklist',
+ path: 'commands/cmd.yaml',
+ message: 'deleted the commands file',
+ sha: await getSha(payload.pull_request.head.sha, 'commands', 'cmd.yaml'),
+ branch: 'master'
+ }
+ await github.repos.deleteFile(commandsOptions);
+ } else {
+ debug('AutoCommit is turned off - Continuing');
+ }
+ } catch (e) {
+ debug(e);
+ }
+ }
+ }
+ }
+})
process.on('unhandledRejection', reason => {
debug(reason);
diff --git a/config/config.example.json b/config/config.example.json
index e8476db..b0af79e 100644
--- a/config/config.example.json
+++ b/config/config.example.json
@@ -2,5 +2,9 @@
"port": 80,
"webhookSecret": "AbcDeFGHiJKLMn12345",
"githubAppID": "1",
- "urlScanAPIKey": "abcdefgh-01234-5678-ijkl-mnopqrstuvwx"
+ "urlScanAPIKey": "abcdefgh-01234-5678-ijkl-mnopqrstuvwx",
+ "githubAccessKey": "0000a000000a0000000a0000000a000000000",
+ "makeComment": true,
+ "autoCommit": true,
+ "deleteCommands": true
}
\ No newline at end of file
diff --git a/package.json b/package.json
index 3d55d72..2003bfe 100644
--- a/package.json
+++ b/package.json
@@ -17,12 +17,15 @@
"url": "git://github.com/CryptoScamDB/github.git"
},
"dependencies": {
+ "@octokit/webhooks": "^5.3.1",
"bottleneck": "^2.8.0",
"cross-env": "^5.2.0",
"debug": "^3.1.0",
+ "download": "^7.1.0",
+ "fs": "0.0.1-security",
"github-app": "^4.0.1",
- "github-webhook-handler": "^0.7.1",
"js-yaml": "^3.12.0",
+ "octonode": "^0.9.5",
"request": "^2.88.0"
},
"devDependencies": {
diff --git a/utils/createComment.js b/utils/createComment.js
new file mode 100644
index 0000000..d77a655
--- /dev/null
+++ b/utils/createComment.js
@@ -0,0 +1,177 @@
+const debug = require('debug')('createContent');
+const yaml = require('js-yaml');
+const download = require('download')
+const urlScanReport = require('./urlscan');
+const app = require('./github');
+const server = require('./server');
+const config = require('./config');
+const makeReq = require('./makeReq');
+
+
+const createComment = (payload) => {
+ return new Promise(async (resolve, reject) => {
+ debug("New PR opened! (" + + "/" + payload.repository.name + "; #" + payload.pull_request.number + ")");
+ const github = await app.asInstallation(payload.installation.id);
+
+ /* CommandsFile */
+ let originalBranchCommands
+ let pullRequestBranchCommands;
+ try {
+ originalBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/commands/cmd.yaml');
+ pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml');
+ } catch (e) {
+ debug("Getting PR branch scams...");
+ if(!originalBranchCommands) {
+ originalBranchCommands = [];
+ }
+ if(!pullRequestBranchCommands) {
+ try {
+ pullRequestBranchCommands = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.head.sha + '/commands/cmd.yaml');
+ } catch (e) {
+ pullRequestBranchCommands = [];
+ }
+ }
+ }
+
+ /* Handle Commands Additions */
+ let newCommandsEntries;
+ if(!originalBranchCommands && !pullRequestBranchCommands) {
+ newCommandsEntries = null;
+ } else {
+ const originalCommandsContent = yaml.safeLoad(Buffer.from(originalBranchCommands,'base64').toString());
+ const pullRequestCommandsContent = yaml.safeLoad(Buffer.from(pullRequestBranchCommands,'base64').toString());
+ if(originalCommandsContent && pullRequestCommandsContent) {
+ const oldCommandsEntries = originalCommandsContent.map(entry => entry.data.url);
+ newCommandsEntries = await Promise.all(
+ pullRequestCommandsContent.map(
+ entry => entry.data.url
+ ).filter(
+ entry => !oldCommandsEntries.includes(entry)
+ ).map(
+ url => pullRequestCommandsContent.find(
+ entry => entry.data.url === url
+ )
+ ).map(async entry => {
+ entry.data.URLScan = (await urlScanReport(entry.data.url)) || '(Error)';
+ return entry.data;
+ })
+ );
+ } else if(!originalCommandsContent && pullRequestCommandsContent){
+ if(!pullRequestCommandsContent.length) {
+ newCommandsEntries = [pullRequestCommandsContent.data];
+ } else {
+ newCommandsEntries = pullRequestCommandsContent.map(entry => {
+ if (entry.type.toLowerCase() === "ADD".toLowerCase()) {
+ return entry.data;
+ }
+ })
+ }
+ }
+
+ }
+
+ /* Combine ScamsFile and Commands Additions */
+ debug("Combining ScamsFile and Commands Additions...");
+ const newEntriesArray = [];
+ await Promise.all(
+ newCommandsEntries.map(entry => {
+ newEntriesArray.push(entry);
+ })
+ );
+ const newEntriesConst = newEntriesArray;
+
+ /* Download scams file */
+ const originalScamsFile = await download('https://raw.githubusercontent.com/CryptoScamDB/blacklist/' + payload.pull_request.base.sha + '/data/urls.yaml');
+ const parsedOriginalScamsFile = await yaml.safeLoad(Buffer.from(originalScamsFile,'base64').toString());
+ const newScamsFile = await JSON.parse(JSON.stringify(parsedOriginalScamsFile));
+ await Promise.all(
+ newEntriesConst.map(entry => {
+ newScamsFile.push(entry)
+ })
+ );
+
+ /* Create new commit */
+ const newScamsMaterial = await Buffer.from(yaml.safeDump(newScamsFile, { lineWidth: 99999999, indent: 4 })).toString('base64');
+ const headUrl = 'https://api.github.com/repos/cryptoscamdb/blacklist/git/trees/' + payload.pull_request.head.sha;
+ const allTreesPreAdd = JSON.parse(await makeReq(headUrl));
+ const dataTreeItemPreAdd = await allTreesPreAdd.tree.find(entry => {
+ return entry.path === 'data';
+ });
+ const allDataTrees = JSON.parse(await makeReq(dataTreeItemPreAdd.url));
+ const urlsObject = await allDataTrees.tree.find(entry => {
+ return (entry.path === 'urls.yaml')
+ });
+
+ let options = {
+ owner: 'cryptoscamdb',
+ repo: 'blacklist',
+ path: 'data/urls.yaml',
+ message: 'Added new entry',
+ content: newScamsMaterial,
+ sha: urlsObject.sha,
+ branch: payload.pull_request.head.ref
+ }
+
+ try {
+ if (config.autoCommit) {
+ debug('AutoCommit is still in development - Continuing');
+ } else {
+ debug('AutoCommit is still in development and turned off - Continuing');
+ }
+ } catch (e) {
+ debug(e);
+ }
+
+ /* Add URLScan to the message */
+ const newEntries = await Promise.all(
+ newEntriesArray.map(async entry => {
+ entry.URLScan = (await urlScanReport(entry.url)) || '(Error)';
+ return entry;
+ })
+ );
+ let duplicate = false;
+ const duplicateEntries = [];
+ await Promise.all(
+ await newEntries.map(async entry => {
+ parsedOriginalScamsFile.map(en => {
+ if (en.url.toLowerCase() === entry.url.toLowerCase()) {
+ duplicateEntries.push(entry);
+ };
+ });
+ })
+ );
+
+ duplicate = duplicateEntries.length >= 1 ? true : false;
+
+ /* Creating comment with new additions */
+ if(newEntries.length > 0) {
+ debug('Making Comments now')
+ if (!duplicate) {
+ await github.issues.createComment({
+ owner: payload.repository.owner.login,
+ repo: payload.repository.name,
+ number: payload.pull_request.number,
+ body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n")
+ });
+ } else {
+ await github.issues.createComment({
+ owner: payload.repository.owner.login,
+ repo: payload.repository.name,
+ number: payload.pull_request.number,
+ body: '**New entries added**: \n\n' + newEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n") + '\n\n' + '**Duplicate entries detected**: \n\n' + duplicateEntries.map(entry => Object.keys(entry).map(key => '**' + key + '**: ' + entry[key]).join('\n')).join("\n
\n")
+ });
+ }
+
+ } else {
+ await github.issues.createComment({
+ owner: payload.repository.owner.login,
+ repo: payload.repository.name,
+ number: payload.pull_request.number,
+ body: '**No new entries added**'
+ });
+ }
+ debug("Done!");
+ })
+}
+
+module.exports = createComment;
\ No newline at end of file
diff --git a/utils/getSha.js b/utils/getSha.js
new file mode 100644
index 0000000..6ad65c1
--- /dev/null
+++ b/utils/getSha.js
@@ -0,0 +1,24 @@
+const makeReq = require('./makeReq');
+const debug = require('debug')('getSha');
+
+const getSha = (headSha, folder, file) => {
+ return new Promise(async (resolve, reject) => {
+ try {
+ const headUrl = 'https://api.github.com/repos/cryptoscamdb/blacklist/git/trees/' + headSha;
+ const allTreesPreAdd = JSON.parse(await makeReq(headUrl));
+ const dataTreeItemPreAdd = await allTreesPreAdd.tree.find(entry => {
+ return (entry.path === folder);
+ });
+ const allDataTrees = JSON.parse(await makeReq(dataTreeItemPreAdd.url));
+ const urlsObject = await allDataTrees.tree.find(entry => {
+ return (entry.path === file);
+ });
+ resolve(urlsObject.sha);
+ } catch(err) {
+ debug(err);
+ reject(err);
+ }
+ })
+}
+
+module.exports = getSha;
\ No newline at end of file
diff --git a/utils/github.js b/utils/github.js
index facc25f..b8638c2 100644
--- a/utils/github.js
+++ b/utils/github.js
@@ -3,8 +3,18 @@ const createGitHubApp = require('github-app');
const config = require('./config');
const privateKey = require('./privatekey');
-debug("Registering Github app...");
-module.exports = createGitHubApp({
- id: config.githubAppID,
- cert: privateKey
-});
\ No newline at end of file
+
+const createGHApp = () => {
+ try {
+ debug("Registering Github app...");
+ return (createGitHubApp({
+ id: config.githubAppID,
+ cert: privateKey
+ }));
+ debug("Finished registering Github app.");
+ } catch (err) {
+ debug(err);
+ }
+}
+
+module.exports = createGHApp();
\ No newline at end of file
diff --git a/utils/makeReq.js b/utils/makeReq.js
new file mode 100644
index 0000000..4b91d6b
--- /dev/null
+++ b/utils/makeReq.js
@@ -0,0 +1,20 @@
+const request = require('request');
+
+const makeRequest = (url) => {
+ return new Promise(async (resolve, reject) => {
+ let options = {
+ method: "GET",
+ headers: {
+ "User-Agent": 'scamreportbot'
+ }
+ }
+ request(url, options, (err, response, body) => {
+ if(err || response.statusCode === 404 || response.statusCode === 403) reject(err);
+ else {
+ resolve(body);
+ }
+ })
+ })
+}
+
+module.exports = makeRequest;
\ No newline at end of file
diff --git a/utils/server.js b/utils/server.js
index 1670348..9ceacd8 100644
--- a/utils/server.js
+++ b/utils/server.js
@@ -1,18 +1,7 @@
const debug = require('debug')('http');
const http = require('http');
const config = require('./config');
-const webhook = require('./webhook');
+const webhooks = require('./webhook');
debug("Registering http server...");
-module.exports = http.createServer((req, res) => {
- webhook(req, res, err => {
- if (err) {
- console.error(err);
- res.statusCode = 500;
- res.end('500');
- } else {
- res.statusCode = 404;
- res.end('404');
- }
- });
-}).listen(config.port);
\ No newline at end of file
+module.exports = http.createServer(webhooks.middleware).listen(config.port);
\ No newline at end of file
diff --git a/utils/webhook.js b/utils/webhook.js
index 430aba1..fac1748 100644
--- a/utils/webhook.js
+++ b/utils/webhook.js
@@ -1,9 +1,18 @@
const debug = require('debug')('webhook');
-const createWebhook = require('github-webhook-handler');
+const createWebhook = require('@octokit/webhooks');
const config = require('./config');
-debug("Registering webhook...");
-module.exports = createWebhook({
- path: '/',
- secret: config.webhookSecret
-});
\ No newline at end of file
+const createGHWebhook = () => {
+ try {
+ debug("Registering webhook...");
+ return (new createWebhook({
+ secret: config.webhookSecret
+ }));
+ debug("Webhook registered")
+ } catch (err) {
+ debug(err);
+ }
+}
+
+
+module.exports = createGHWebhook();
\ No newline at end of file