getting very close

browser/main/lib/dataApi/createNoteFromUrl.js

@@ -0,0 +1,94 @@
+const http = require('http')
+const https = require('https')
+const TurndownService = require('turndown')
+const createNote = require('./createNote')
+
+import { hashHistory } from 'react-router'
+import ee from 'browser/main/lib/eventEmitter'
+
+function validateUrl(str) {
+  if(/^(?:(?:(?:https?|ftp):)?\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})).?)(?::\d{2,5})?(?:[/?#]\S*)?$/i.test(str)) {
+    return true;
+  } else {
+    return false;
+  }
+}
+
+function createNoteFromUrl (url, storage, folder, dispatch = null, location = null) {
+  return new Promise((resolve, reject) => {
+    let td = new TurndownService();
+
+    if(!validateUrl(url)) {
+      reject({result: false, error: "Please check your URL is in correct format. (Example, https://www.google.com)"})
+    }
+
+    let request = http
+    if(url.includes('https')) {
+      request = https
+    }
+
+    let req = request.request(url, (res) => {
+      let data = ''
+
+      res.on('data', (chunk) => {
+        data += chunk
+      })
+
+      res.on('end', () => {
+        let html = document.createElement('html')
+        html.innerHTML = data
+
+        let scripts = html.getElementsByTagName('script')
+        for(let i = scripts.length - 1; i >= 0; i--) {
+          scripts[i].parentNode.removeChild(scripts[i])
+        }
+
+        let body = html.getElementsByTagName('body')[0].innerHTML
+        let markdownHTML = td.turndown(body)
+
+        html.innerHTML = ''
+
+        if(dispatch !== null) {
+          createNote(storage, {
+            type: 'MARKDOWN_NOTE',
+            folder: folder,
+            title: '',
+            content: markdownHTML
+          })
+          .then((note) => {
+            const noteHash = note.key
+            dispatch({
+              type: 'UPDATE_NOTE',
+              note: note
+            })
+            hashHistory.push({
+              pathname: location.pathname,
+              query: {key: noteHash}
+            })
+            ee.emit('list:jump', noteHash)
+            ee.emit('detail:focus')
+            resolve({result: true, error: null})
+          })
+        } else {
+          createNote(storage, {
+            type: 'MARKDOWN_NOTE',
+            folder: folder,
+            title: '',
+            content: markdownHTML
+          }).then((note) => {
+            resolve({result: true, error: null})
+          })
+        }
+      })
+    })
+
+    req.on('error', (e) => {
+      console.error('error in parsing URL', e)
+      reject({result: false, error: e})
+    })
+    req.end();
+  })
+
+}
+
+module.exports = createNoteFromUrl;

browser/main/lib/dataApi/index.js

@@ -9,6 +9,7 @@ const dataApi = {
   reorderFolder: require('./reorderFolder'),
   exportFolder: require('./exportFolder'),
   createNote: require('./createNote'),
+  createNoteFromUrl: require('./createNoteFromUrl'),
   updateNote: require('./updateNote'),
   deleteNote: require('./deleteNote'),
   moveNote: require('./moveNote'),

browser/main/modals/CreateMarkdownFromURLModal.js

@@ -8,22 +8,15 @@ import consts from 'browser/lib/consts'
 import ModalEscButton from 'browser/components/ModalEscButton'
 import AwsMobileAnalyticsConfig from 'browser/main/lib/AwsMobileAnalyticsConfig'
 import i18n from 'browser/lib/i18n'
-const http = require('http')
-const https = require('https')
-const TurndownService = require('turndown')
-import { hashHistory } from 'react-router'
-import ee from 'browser/main/lib/eventEmitter'
 
 class CreateMarkdownFromURLModal extends React.Component {
   constructor (props) {
     super(props)
-    let td = new TurndownService();
 
     this.state = {
       name: '',
       showerror: false,
-      errormessage: '',
+      errormessage: ''
-      turndownService: td
     }
 
   }
@@ -60,87 +53,29 @@ class CreateMarkdownFromURLModal extends React.Component {
     this.confirm()
   }
 
-  showError(message) {
+  showError (message) {
     this.setState({
       showerror: true,
       errormessage: message
     });
   }
 
-  hideError() {
+  hideError () {
     this.setState({
       showerror: false,
       errormessage: ''
     })
   }
 
-  validateUrl(str) {
-    if(/^(?:(?:(?:https?|ftp):)?\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})).?)(?::\d{2,5})?(?:[/?#]\S*)?$/i.test(str)) {
-      return true;
-    } else {
-      return false;
-    }
-  }
-
   confirm () {
-    if(this.validateUrl(this.state.name)) {
       this.hideError()
-      let url = this.state.name;
+      const { storage, folder, dispatch, location } = this.props
-      let request = http;
-      if(url.includes('https'))
-        request = https;
-      let req = request.request(url, (res) => {
-        let data = '';
-        res.on('data', (chunk) => {
-          data += chunk
-        })
-        res.on('end', () => {
-          console.log("receiving data", data);
 
-          let html = document.createElement('html');
+      let note = dataApi.createNoteFromUrl(this.state.name, storage, folder, dispatch, location).then((result) => {
-          html.innerHTML = data;
+        this.props.close()
-
+      }).catch((result) => {
-          let scripts = html.getElementsByTagName('script');
+        this.showError(result.error);
-          for(let i = scripts.length - 1; i >= 0; i--) {
-            scripts[i].parentNode.removeChild(scripts[i]);
-          }
-
-          let body = html.getElementsByTagName('body')[0].innerHTML;
-          let markdownHTML = this.state.turndownService.turndown(body);
-          console.log('markdown', markdownHTML);
-          html.innerHTML = '';
-          const { storage, folder, dispatch, location } = this.props
-
-          dataApi
-            .createNote(storage, {
-              type: 'MARKDOWN_NOTE',
-              folder: folder,
-              title: '',
-              content: markdownHTML
-            })
-            .then((note) => {
-              const noteHash = note.key
-              dispatch({
-                type: 'UPDATE_NOTE',
-                note: note
-              })
-              hashHistory.push({
-                pathname: location.pathname,
-                query: {key: noteHash}
-              })
-              ee.emit('list:jump', noteHash)
-              ee.emit('detail:focus')
-              this.props.close()
-            })
-        })
       });
-      req.on('error', (e) => {
-        console.log("Error in request", e.message);
-      });
-      req.end();
-    } else {
-      this.showError("Please check your URL is in correct format. (Example, 'https://google.com')")
-    }
   }
 
   render () {

tests/dataApi/createNoteFromUrl-test.js

@@ -0,0 +1,54 @@
+const test = require('ava')
+const createNoteFromUrl = require('browser/main/lib/dataApi/createNoteFromUrl')
+
+global.document = require('jsdom').jsdom('<body></body>')
+global.window = document.defaultView
+global.navigator = window.navigator
+
+const Storage = require('dom-storage')
+const localStorage = window.localStorage = global.localStorage = new Storage(null, { strict: true })
+const path = require('path')
+const TestDummy = require('../fixtures/TestDummy')
+const sander = require('sander')
+const os = require('os')
+const CSON = require('@rokt33r/season')
+const faker = require('faker')
+
+const storagePath = path.join(os.tmpdir(), 'test/create-note-from-url')
+
+test.beforeEach((t) => {
+  t.context.storage = TestDummy.dummyStorage(storagePath)
+  localStorage.setItem('storages', JSON.stringify([t.context.storage.cache]))
+})
+
+test.serial('Create a note from URL', (t) => {
+  const storageKey = t.context.storage.cache.key
+  const folderKey = t.context.storage.json.folders[0].key
+
+  const url = 'https://shapeshed.com/writing-cross-platform-node/'
+
+
+  return Promise.resolve()
+    .then(function doTest () {
+      return Promise.all([
+        createNoteFromUrl(url, storageKey, folderKey)
+      ])
+    })
+    .then(function assert (data) {
+      const data1 = data[0]
+
+      console.log("STORM LOOK HERE", data1)
+
+      t.is(storageKey, data1.storage)
+      const jsonData2 = CSON.readFileSync(path.join(storagePath, 'notes', data1.key + '.cson'))
+      t.is(input2.content, data2.content)
+      t.is(input2.content, jsonData2.content)
+      t.is(input2.tags.length, data2.tags.length)
+      t.is(input2.tags.length, jsonData2.tags.length)
+    })
+})
+
+test.after(function after () {
+  localStorage.clear()
+  sander.rimrafSync(storagePath)
+})

tests/lib/html-to-md-test.js

@@ -1,2 +0,0 @@
-const test = require('ava')
-const htmlToMd = require('browser/main/modals/CreateMarkdownFromURLModal')

yarn.lock

@@ -2301,6 +2301,12 @@ cssom@0.3.x, "cssom@>= 0.3.2 < 0.4.0":
   dependencies:
     cssom "0.3.x"
 
+"cssstyle@>= 0.3.1 < 0.4.0":
+  version "0.3.1"
+  resolved "https://registry.yarnpkg.com/cssstyle/-/cssstyle-0.3.1.tgz#6da9b4cff1bc5d716e6e5fe8e04fcb1b50a49adf"
+  dependencies:
+    cssom "0.3.x"
+
 ctype@0.5.3:
   version "0.5.3"
   resolved "https://registry.yarnpkg.com/ctype/-/ctype-0.5.3.tgz#82c18c2461f74114ef16c135224ad0b9144ca12f"
@@ -2331,6 +2337,14 @@ dashdash@^1.12.0:
   dependencies:
     assert-plus "^1.0.0"
 
+data-urls@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-1.0.0.tgz#24802de4e81c298ea8a9388bb0d8e461c774684f"
+  dependencies:
+    abab "^1.0.4"
+    whatwg-mimetype "^2.0.0"
+    whatwg-url "^6.4.0"
+
 date-fns@^1.23.0:
   version "1.28.5"
   resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.28.5.tgz#257cfc45d322df45ef5658665967ee841cd73faf"
@@ -5249,6 +5263,37 @@ jsdom@11.6.2, jsdom@^11.5.1:
     ws "^4.0.0"
     xml-name-validator "^3.0.0"
 
+jsdom@^11.9.0:
+  version "11.11.0"
+  resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-11.11.0.tgz#df486efad41aee96c59ad7a190e2449c7eb1110e"
+  dependencies:
+    abab "^1.0.4"
+    acorn "^5.3.0"
+    acorn-globals "^4.1.0"
+    array-equal "^1.0.0"
+    cssom ">= 0.3.2 < 0.4.0"
+    cssstyle ">= 0.3.1 < 0.4.0"
+    data-urls "^1.0.0"
+    domexception "^1.0.0"
+    escodegen "^1.9.0"
+    html-encoding-sniffer "^1.0.2"
+    left-pad "^1.2.0"
+    nwsapi "^2.0.0"
+    parse5 "4.0.0"
+    pn "^1.1.0"
+    request "^2.83.0"
+    request-promise-native "^1.0.5"
+    sax "^1.2.4"
+    symbol-tree "^3.2.2"
+    tough-cookie "^2.3.3"
+    w3c-hr-time "^1.0.1"
+    webidl-conversions "^4.0.2"
+    whatwg-encoding "^1.0.3"
+    whatwg-mimetype "^2.1.0"
+    whatwg-url "^6.4.1"
+    ws "^4.0.0"
+    xml-name-validator "^3.0.0"
+
 jsdom@^9.4.2:
   version "9.12.0"
   resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-9.12.0.tgz#e8c546fffcb06c00d4833ca84410fed7f8a097d4"
@@ -6259,6 +6304,10 @@ nwmatcher@^1.4.3:
   version "1.4.3"
   resolved "https://registry.yarnpkg.com/nwmatcher/-/nwmatcher-1.4.3.tgz#64348e3b3d80f035b40ac11563d278f8b72db89c"
 
+nwsapi@^2.0.0:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.0.1.tgz#a50d59a2dcb14b6931401171713ced2d0eb3468f"
+
 oauth-sign@~0.6.0:
   version "0.6.0"
   resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.6.0.tgz#7dbeae44f6ca454e1f168451d630746735813ce3"
@@ -8570,7 +8619,7 @@ tough-cookie@>=2.3.3, tough-cookie@^2.3.3, tough-cookie@~2.3.3:
   dependencies:
     punycode "^1.4.1"
 
-tr46@^1.0.0:
+tr46@^1.0.0, tr46@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-1.0.1.tgz#a8b13fd6bfd2489519674ccde55ba3693b706d09"
   dependencies:
@@ -8624,6 +8673,12 @@ tunnel-agent@~0.4.0:
   version "0.4.3"
   resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb"
 
+turndown@^4.0.2:
+  version "4.0.2"
+  resolved "https://registry.yarnpkg.com/turndown/-/turndown-4.0.2.tgz#c3ddb8ba32a3665723599be2f4e7860adb6042ae"
+  dependencies:
+    jsdom "^11.9.0"
+
 tweetnacl@^0.14.3, tweetnacl@~0.14.0:
   version "0.14.5"
   resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64"
@@ -9042,6 +9097,10 @@ whatwg-fetch@>=0.10.0:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/whatwg-fetch/-/whatwg-fetch-2.0.3.tgz#9c84ec2dcf68187ff00bc64e1274b442176e1c84"
 
+whatwg-mimetype@^2.0.0, whatwg-mimetype@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.1.0.tgz#f0f21d76cbba72362eb609dbed2a30cd17fcc7d4"
+
 whatwg-url@^4.3.0:
   version "4.8.0"
   resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-4.8.0.tgz#d2981aa9148c1e00a41c5a6131166ab4683bbcc0"
@@ -9057,6 +9116,14 @@ whatwg-url@^6.4.0:
     tr46 "^1.0.0"
     webidl-conversions "^4.0.1"
 
+whatwg-url@^6.4.1:
+  version "6.4.1"
+  resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-6.4.1.tgz#fdb94b440fd4ad836202c16e9737d511f012fd67"
+  dependencies:
+    lodash.sortby "^4.7.0"
+    tr46 "^1.0.1"
+    webidl-conversions "^4.0.2"
+
 when@~3.6.x:
   version "3.6.4"
   resolved "https://registry.yarnpkg.com/when/-/when-3.6.4.tgz#473b517ec159e2b85005497a13983f095412e34e"