Add gitlab documentation api parser, add models, some logic changes

generator/crawler.js

@@ -0,0 +1,137 @@
+'use strict';
+
+const jsdom = require('node-jsdom'); // eslint-disable-line
+const fs = require('fs');
+
+const jquery = fs.readFileSync('./jquery.js', 'utf-8');
+// const test = fs.readFileSync('./test.html', 'utf-8');
+const completeGitlabAPI = [];
+
+
+function enterResource(url, callback) {
+    return jsdom.env({
+        url,
+        src: [jquery],
+        done: function (errors, window) {
+            const allEntries = [];
+            const { $ } = window;
+            console.log('Reading resource...');
+            $(':header').each(function () {
+                console.log(' ');
+                let title = $(this).attr('id');
+                if (!title) {
+                    return;
+                }
+                title = title.replace('-39-s', '');
+
+                if (title === 'help-and-feedback') {
+                    return;
+                }
+
+                console.log('############# New Entry ##############');
+                const description = $(this).nextAll('p').first().text();
+                const api = $(this).nextAll('div').first().find('pre')
+                    .text()
+                    .split('\n')
+                    .filter(e => e);
+
+                console.log(`Title: ${title}`);
+                console.log(`Description: ${description}`);
+                console.log(`api: ${api}`);
+                const entry = {
+                    title,
+                    description,
+                    api,
+                    params: [],
+                };
+                allEntries.push(entry);
+
+                let currentNode = $(this);
+                let skipCounter = 0;
+                while (!currentNode.is('table') && !currentNode.is('ul') && skipCounter < 7) {
+                    currentNode = currentNode.next();
+                    skipCounter += 1;
+                }
+
+                if (currentNode.is('table')) {
+                    $(this).nextAll('table').first().find('tbody')
+                        .children()
+                        .each(function () {
+                            // table entries
+                            console.log('-------- New Line.------------');
+                            const tableEntries = ['attribute', 'type', 'required', 'description'];
+                            let counter = 0;
+                            const paramEntry = {};
+                            entry.params.push(paramEntry);
+
+                            // One entry
+                            $(this).children().each(function () {
+                                let singleEntry = $(this).text();
+
+                                if ($(this).text() === '') {
+                                    singleEntry = $(this).find('code').text();
+                                }
+
+                                console.log(`${tableEntries[counter]}: ${singleEntry}`);
+                                paramEntry[tableEntries[counter]] = singleEntry;
+
+                                counter += 1;
+                            });
+                        });
+                } else if (currentNode.is('ul')) {
+                    const listText = currentNode.children().first();
+                    const attribute = listText.find('code').text();
+                    const type = '';
+                    const required = listText.text().indexOf('required') !== -1 ? 'required' : 'optional';
+                    const paramDescription = listText.text().split('-')[1];
+
+                    const paramEntry = {
+                        attribute,
+                        type,
+                        required,
+                        paramDescription,
+                    };
+
+                    entry.params.push(paramEntry);
+                }
+            });
+
+            return callback(allEntries);
+        },
+    });
+}
+
+return jsdom.env({
+    url: 'https://docs.gitlab.com/ee/api/README.html',
+    src: [jquery],
+    done: function (errors, window) {
+        const { $ } = window;
+        console.log('Links');
+        // Get List of entries.
+        $('#resources').next().next().children()
+            .each(function () {
+                const urlPart = $(this).children(':first').attr('href');
+                console.log(urlPart);
+
+                if (urlPart === 'v3_to_v4.html') {
+                    return;
+                }
+
+                if (urlPart === '../ci/triggers/README.html') {
+                    return;
+                }
+
+                const url = `https://docs.gitlab.com/ee/api/${urlPart}`;
+                enterResource(url, (allEntries) => {
+                    const apiType = {
+                        url,
+                        urlPart,
+                        allEntries,
+                    };
+
+                    completeGitlabAPI.push(apiType);
+                    fs.writeFileSync('./result.json', JSON.stringify(completeGitlabAPI), 'utf-8');
+                });
+            });
+    },
+});

generator/fixMoreThanOneApiDefined.js

@@ -0,0 +1,62 @@
+'use strict';
+
+const jsonDef = require('../result_21-11-18-fixed.json');
+const fs = require('fs');
+// const JsDiff = require('diff');
+
+const newResult = [];
+
+jsonDef.forEach((api) => {
+    const newEntrys = {};
+    newResult.push(newEntrys);
+    newEntrys.url = api.url;
+    newEntrys.urlPart = api.urlPart;
+    newEntrys.allEntries = [];
+
+    api.allEntries.forEach((entry) => {
+        entry.api = entry.api.filter(e => e.indexOf('?') === -1 && e.indexOf('=>') === -1);
+
+        if (entry.api.length === 0) {
+            console.error(`No api defined for title: ${entry.title}`);
+
+            return;
+        }
+
+        if (entry.api.length === 1) {
+            newEntrys.allEntries.push(entry);
+
+            return;
+        }
+        console.log('----------------------------');
+        console.log(`More than one api defined for title: ${entry.title} Rewrite titles`);
+
+        const functionNames = [];
+        for (let i = 0; i < entry.api.length; i++) {
+            const splitO = entry.api[i].split('/');
+            for (let k = 0; k < entry.api.length; k++) {
+                const splitI = entry.api[k].split('/');
+
+                for (let s = 0; s < splitI.length; s++) {
+                    if (splitO.length > s && splitO[s] !== splitI[s]) {
+                        let functionName = `${entry.title}-${splitI[s]}`;
+                        functionName = functionName.replace(/_/g, '');
+                        if (functionName.indexOf(':') === -1 && functionNames.indexOf(functionName) === -1) {
+                            functionNames.push(functionName);
+                            console.log(`Add function: ${functionName}`);
+                            const entryCopy = JSON.parse(JSON.stringify(entry));
+                            entryCopy.api = [];
+                            entryCopy.api.push(entry.api[k]);
+                            console.log(`With api: ${entry.api[k]}`);
+                            entryCopy.title = functionName;
+
+                            newEntrys.allEntries.push(entryCopy);
+                        }
+                    }
+                }
+            }
+        }
+    });
+});
+
+fs.writeFileSync('result-fixed.json', JSON.stringify(newResult));
+