diff --git a/.gitignore b/.gitignore index 0098ccd9..1cc93beb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,9 @@ node_modules/ output/ tmp/ +test/ #ignore specific filenames export.xml .DS_Store +*.xml diff --git a/src/parser.js b/src/parser.js index 14514853..65f0a002 100644 --- a/src/parser.js +++ b/src/parser.js @@ -5,139 +5,234 @@ const xml2js = require('xml2js'); const shared = require('./shared'); const settings = require('./settings'); const translator = require('./translator'); +const titleCase = require("./titlecase"); async function parseFilePromise(config) { - console.log('\nParsing...'); - const content = await fs.promises.readFile(config.input, 'utf8'); - const data = await xml2js.parseStringPromise(content, { - trim: true, - tagNameProcessors: [xml2js.processors.stripPrefix] - }); - - const postTypes = getPostTypes(data, config); - const posts = collectPosts(data, postTypes, config); - - const images = []; - if (config.saveAttachedImages) { - images.push(...collectAttachedImages(data)); - } - if (config.saveScrapedImages) { - images.push(...collectScrapedImages(data, postTypes)); - } - - mergeImagesIntoPosts(images, posts); - - return posts; + console.log("\nParsing..."); + const content = await fs.promises.readFile(config.input, "utf8"); + const data = await xml2js.parseStringPromise(content, { + trim: true, + tagNameProcessors: [xml2js.processors.stripPrefix], + }); + + const postTypes = getPostTypes(data, config); + const posts = collectPosts(data, postTypes, config); + + const images = []; + if (config.saveAttachedImages) { + images.push(...collectAttachedImages(data)); + } + if (config.saveScrapedImages) { + images.push(...collectScrapedImages(data, postTypes)); + } + + mergeImagesIntoPosts(images, posts); + + return posts; } function getPostTypes(data, config) { - if (config.includeOtherTypes) { - // search export file for all post types minus some default types we don't want - // effectively this will be 'post', 'page', and custom post types - const types = data.rss.channel[0].item - .map(item => item.post_type[0]) - .filter(type => !['attachment', 'revision', 'nav_menu_item', 'custom_css', 'customize_changeset'].includes(type)); - return [...new Set(types)]; // remove duplicates - } else { - // just plain old vanilla "post" posts - return ['post']; - } + if (config.includeOtherTypes) { + // search export file for all post types minus some default types we don't want + // effectively this will be 'post', 'page', and custom post types + const types = data.rss.channel[0].item + .map((item) => item.post_type[0]) + .filter( + (type) => + ![ + "attachment", + "revision", + "nav_menu_item", + "custom_css", + "customize_changeset", + "acf-field", + "acf-field-group", + "elementor_template", + "elementor_library", + "schema", + "fl-builder-template", + "flamingo_contact", + "flamingo_inbound", + "wpcf7_contact_form", + "generate_page_header", + ].includes(type) + ); + return [...new Set(types)]; // remove duplicates + } else { + // just plain old vanilla "post" posts + return ["post"]; + } } function getItemsOfType(data, type) { - return data.rss.channel[0].item.filter(item => item.post_type[0] === type); + return data.rss.channel[0].item.filter((item) => item.post_type[0] === type); } function collectPosts(data, postTypes, config) { - // this is passed into getPostContent() for the markdown conversion - const turndownService = translator.initTurndownService(); - - let allPosts = []; - postTypes.forEach(postType => { - const postsForType = getItemsOfType(data, postType) - .filter(post => post.status[0] !== 'trash' && post.status[0] !== 'draft') - .map(post => ({ - // meta data isn't written to file, but is used to help with other things - meta: { - id: getPostId(post), - slug: getPostSlug(post), - coverImageId: getPostCoverImageId(post), - type: postType, - imageUrls: [] - }, - frontmatter: { - title: getPostTitle(post), - date: getPostDate(post), - categories: getCategories(post), - tags: getTags(post) - }, - content: translator.getPostContent(post, turndownService, config) - })); - - if (postTypes.length > 1) { - console.log(`${postsForType.length} "${postType}" posts found.`); - } - - allPosts.push(...postsForType); - }); - - if (postTypes.length === 1) { - console.log(allPosts.length + ' posts found.'); - } - return allPosts; + // this is passed into getPostContent() for the markdown conversion + const turndownService = translator.initTurndownService(); + + let allPosts = []; + postTypes.forEach((postType) => { + const postsForType = getItemsOfType(data, postType) + .filter( + (post) => post.status[0] !== "trash" && post.status[0] !== "draft" + ) + .map((post) => ({ + // meta data isn't written to file, but is used to help with other things + meta: { + id: getPostId(post), + slug: getPostSlug(post), + coverImageId: getPostCoverImageId(post), + type: postType, + imageUrls: [], + }, + frontmatter: { + title: getPostTitle(post), + slug: getPostSlug(post), + description: getSeoDescription(post), + date: getPostDate(post), + keywords: getSeoKeywords(post), + categories: getCategories(post), + tags: getTags(post), + datePublishedGmt: getDateGmt(post), + dateModifiedGmt: getModifiedGmt(post), + originLink: post.link[0], + postId: getPostId(post), + postParent: getPostParent(post), + }, + content: translator.getPostContent(post, turndownService, config), + })); + + if (postTypes.length > 1) { + console.log(`${postsForType.length} "${postType}" posts found.`); + } + + allPosts.push(...postsForType); + }); + + if (postTypes.length === 1) { + console.log(allPosts.length + " posts found."); + } + return allPosts; } function getPostId(post) { - return post.post_id[0]; + return post.post_id[0]; +} +function getPostParent(post) { + return post.post_parent[0]; } function getPostSlug(post) { - return decodeURIComponent(post.post_name[0]); + return decodeURIComponent(post.post_name[0]); } function getPostCoverImageId(post) { - if (post.postmeta === undefined) { - return undefined; - } - - const postmeta = post.postmeta.find(postmeta => postmeta.meta_key[0] === '_thumbnail_id'); - const id = postmeta ? postmeta.meta_value[0] : undefined; - return id; + if (post.postmeta === undefined) { + return undefined; + } + + const postmeta = post.postmeta.find( + (postmeta) => postmeta.meta_key[0] === "_thumbnail_id" + ); + const id = postmeta ? postmeta.meta_value[0] : undefined; + return id; +} +// get yoast seo meta tags +function getSeoTitle(post) { + if (post.postmeta === undefined) { + return undefined; + } + + const postmeta = post.postmeta.find( + (postmeta) => postmeta.meta_key[0] === "_yoast_wpseo_title" + ); + const id = postmeta ? postmeta.meta_value[0] : undefined; + return id; +} +function getSeoDescription(post) { + if (post.postmeta === undefined) { + return undefined; + } + + const postmeta = post.postmeta.find( + (postmeta) => postmeta.meta_key[0] === "_yoast_wpseo_metadesc" + ); + const id = postmeta ? postmeta.meta_value[0] : undefined; + return id; +} +function getSeoKeywords(post) { + if (post.postmeta === undefined) { + return undefined; + } + + const postmeta = post.postmeta.find( + (postmeta) => postmeta.meta_key[0] === "_yoast_wpseo_focuskw" + ); + const id = postmeta ? postmeta.meta_value[0] : undefined; + return id; } function getPostTitle(post) { - return post.title[0]; + return titleCase(post.title[0]); +} +// function getFrontmatterSlug(post) { +// const url = new URL(post.link[0]); +// return decodeURIComponent(url.pathname.replace(/\//g, '')); +// } +function getPostExcerpt(post) { + const description = post.encoded[1].replace(/(\r\n|\n|\r)/gm, " "); + return description; } function getPostDate(post) { - const dateTime = luxon.DateTime.fromRFC2822(post.pubDate[0], { zone: 'utc' }); - - if (settings.custom_date_formatting) { - return dateTime.toFormat(settings.custom_date_formatting); - } else if (settings.include_time_with_date) { - return dateTime.toISO(); - } else { - return dateTime.toISODate(); - } + const dateTime = luxon.DateTime.fromRFC2822(post.pubDate[0], { + zone: "utc+8", + }); + + if (settings.custom_date_formatting) { + return dateTime.toFormat(settings.custom_date_formatting); + } else if (settings.include_time_with_date) { + return dateTime.toISO(); + } else { + return dateTime.toISODate(); + } } +function getDateGmt(post) { + return post.post_date_gmt[0]; +} +function getModifiedGmt(post) { + if (post.post_modified_gmt === undefined) { + return undefined; + } + return post.post_modified_gmt[0]; +} function getCategories(post) { - const categories = processCategoryTags(post, 'category'); - return categories.filter(category => !settings.filter_categories.includes(category)); + const categories = processCategoryTags(post, "category"); + return categories.filter( + (category) => !settings.filter_categories.includes(category) + ); } function getTags(post) { - return processCategoryTags(post, 'post_tag'); + return processCategoryTags(post, "post_tag"); } function processCategoryTags(post, domain) { - if (!post.category) { - return []; - } - - return post.category - .filter(category => category.$.domain === domain) - .map(({ $: attributes }) => decodeURIComponent(attributes.nicename)); + if (!post.category) { + return []; + } + + return ( + post.category + .filter((category) => category.$.domain === domain) + // .map(({ $: attributes }) => decodeURIComponent(attributes.nicename)); + .map(({ $: attributes }) => + titleCase(attributes.nicename).replace(/-/g, " ") + ) + ); } function collectAttachedImages(data) { diff --git a/src/titlecase.js b/src/titlecase.js new file mode 100644 index 00000000..44f84ebb --- /dev/null +++ b/src/titlecase.js @@ -0,0 +1,126 @@ +const { URL } = require("url"); + +const alwaysLowercase = [ + "a", + "an", + "and", + "at", + "but", + "by", + "for", + "in", + "nor", + "of", + "on", + "or", + "so", + "the", + "to", + "up", + "yet", + "v", + "v.", + "vs", + "vs.", +]; + +const containers = ["(", "[", "{", '"', `'`, "_"]; + +function isUrl(url) { + try { + const parsed = new URL(url); + return Boolean(parsed.hostname); + } catch (error) { + return false; + } +} + +function capitalize(string) { + if (string.length === 0) { + return string; + } + + const letters = [...string]; + const firstLetter = letters.shift(); + + if (containers.indexOf(firstLetter) !== -1) { + return `${firstLetter}${capitalize(letters)}`; + } + + return `${firstLetter.toUpperCase()}${letters.join("")}`; +} + +function titleCase( + string = "", + { excludedWords = [], useDefaultExcludedWords = true } = {} +) { + if (string.toUpperCase() === string) { + string = string.toLowerCase(); + } + + if (useDefaultExcludedWords) { + excludedWords.push(...alwaysLowercase); + } + + const words = string.trim().split(/\s+/); + + const capitalizedWords = words.map((word, index, array) => { + const isFirstWird = index === 0; + const isLastWord = index === words.length - 1; + const isEmail = /.+@.+\..+/.test(word); + const isFilePath = /^(\/[\w.]+)+/.test(word); + const isFileName = /^\w+\.\w{1,3}$/.test(word); + const hasInternalCapital = /(?![-‑–—])[a-z]+[A-Z].*/.test(word); + + const previousWord = index > 1 ? array[index - 1] : ""; + const startOfSubPhrase = index > 1 && [...previousWord].pop() === ":"; + + if ( + isEmail || + isUrl(word) || + isFilePath || + isFileName || + hasInternalCapital + ) { + return word; + } + + const hasHyphen = word.match(/[-‑–—]/g); + if (hasHyphen) { + const isMultiPart = hasHyphen.length > 1; + const [hyphenCharacter] = hasHyphen; + + return word + .split(hyphenCharacter) + .map((subWord) => { + if ( + isMultiPart && + excludedWords.indexOf(subWord.toLowerCase()) !== -1 + ) { + return subWord; + } + + return capitalize(subWord); + }) + .join(hyphenCharacter); + } + + if (word.indexOf("/") !== -1) { + return word.split("/").map(capitalize).join("/"); + } + + if (isFirstWird || isLastWord) { + return capitalize(word); + } + + if (!startOfSubPhrase && excludedWords.indexOf(word.toLowerCase()) !== -1) { + return word.toLowerCase(); + } + + return capitalize(word); + }); + + return capitalizedWords.join(" "); +} + +module.exports = titleCase; diff --git a/src/translator.js b/src/translator.js index 41c110b1..b1509cee 100644 --- a/src/translator.js +++ b/src/translator.js @@ -2,89 +2,101 @@ const turndown = require('turndown'); const turndownPluginGfm = require('turndown-plugin-gfm'); function initTurndownService() { - const turndownService = new turndown({ - headingStyle: 'atx', - bulletListMarker: '-', - codeBlockStyle: 'fenced' - }); - - turndownService.use(turndownPluginGfm.tables); - - // preserve embedded tweets - turndownService.addRule('tweet', { - filter: node => node.nodeName === 'BLOCKQUOTE' && node.getAttribute('class') === 'twitter-tweet', - replacement: (content, node) => '\n\n' + node.outerHTML - }); - - // preserve embedded codepens - turndownService.addRule('codepen', { - filter: node => { - // codepen embed snippets have changed over the years - // but this series of checks should find the commonalities - return ( - ['P', 'DIV'].includes(node.nodeName) && - node.attributes['data-slug-hash'] && - node.getAttribute('class') === 'codepen' - ); - }, - replacement: (content, node) => '\n\n' + node.outerHTML - }); - - // preserve embedded scripts (for tweets, codepens, gists, etc.) - turndownService.addRule('script', { - filter: 'script', - replacement: (content, node) => { - let before = '\n\n'; - if (node.previousSibling && node.previousSibling.nodeName !== '#text') { - // keep twitter and codepen