Skip to content

Dev #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
twodft wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Dev #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
38e872b
Add slug to frontmatter
twodft Nov 30, 2021
654362e
add more frontmatter fields
twodft Dec 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@
node_modules/
output/
tmp/
test/

#ignore specific filenames
export.xml
.DS_Store
*.xml
291 changes: 193 additions & 98 deletions src/parser.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,139 +5,234 @@ const xml2js = require('xml2js');
const shared = require('./shared');
const settings = require('./settings');
const translator = require('./translator');
const titleCase = require("./titlecase");

async function parseFilePromise(config) {
console.log('\nParsing...');
const content = await fs.promises.readFile(config.input, 'utf8');
const data = await xml2js.parseStringPromise(content, {
trim: true,
tagNameProcessors: [xml2js.processors.stripPrefix]
});

const postTypes = getPostTypes(data, config);
const posts = collectPosts(data, postTypes, config);

const images = [];
if (config.saveAttachedImages) {
images.push(...collectAttachedImages(data));
}
if (config.saveScrapedImages) {
images.push(...collectScrapedImages(data, postTypes));
}

mergeImagesIntoPosts(images, posts);

return posts;
console.log("\nParsing...");
const content = await fs.promises.readFile(config.input, "utf8");
const data = await xml2js.parseStringPromise(content, {
trim: true,
tagNameProcessors: [xml2js.processors.stripPrefix],
});

const postTypes = getPostTypes(data, config);
const posts = collectPosts(data, postTypes, config);

const images = [];
if (config.saveAttachedImages) {
images.push(...collectAttachedImages(data));
}
if (config.saveScrapedImages) {
images.push(...collectScrapedImages(data, postTypes));
}

mergeImagesIntoPosts(images, posts);

return posts;
}

function getPostTypes(data, config) {
if (config.includeOtherTypes) {
// search export file for all post types minus some default types we don't want
// effectively this will be 'post', 'page', and custom post types
const types = data.rss.channel[0].item
.map(item => item.post_type[0])
.filter(type => !['attachment', 'revision', 'nav_menu_item', 'custom_css', 'customize_changeset'].includes(type));
return [...new Set(types)]; // remove duplicates
} else {
// just plain old vanilla "post" posts
return ['post'];
}
if (config.includeOtherTypes) {
// search export file for all post types minus some default types we don't want
// effectively this will be 'post', 'page', and custom post types
const types = data.rss.channel[0].item
.map((item) => item.post_type[0])
.filter(
(type) =>
![
"attachment",
"revision",
"nav_menu_item",
"custom_css",
"customize_changeset",
"acf-field",
"acf-field-group",
"elementor_template",
"elementor_library",
"schema",
"fl-builder-template",
"flamingo_contact",
"flamingo_inbound",
"wpcf7_contact_form",
"generate_page_header",
].includes(type)
);
return [...new Set(types)]; // remove duplicates
} else {
// just plain old vanilla "post" posts
return ["post"];
}
}

function getItemsOfType(data, type) {
return data.rss.channel[0].item.filter(item => item.post_type[0] === type);
return data.rss.channel[0].item.filter((item) => item.post_type[0] === type);
}

function collectPosts(data, postTypes, config) {
// this is passed into getPostContent() for the markdown conversion
const turndownService = translator.initTurndownService();

let allPosts = [];
postTypes.forEach(postType => {
const postsForType = getItemsOfType(data, postType)
.filter(post => post.status[0] !== 'trash' && post.status[0] !== 'draft')
.map(post => ({
// meta data isn't written to file, but is used to help with other things
meta: {
id: getPostId(post),
slug: getPostSlug(post),
coverImageId: getPostCoverImageId(post),
type: postType,
imageUrls: []
},
frontmatter: {
title: getPostTitle(post),
date: getPostDate(post),
categories: getCategories(post),
tags: getTags(post)
},
content: translator.getPostContent(post, turndownService, config)
}));

if (postTypes.length > 1) {
console.log(`${postsForType.length} "${postType}" posts found.`);
}

allPosts.push(...postsForType);
});

if (postTypes.length === 1) {
console.log(allPosts.length + ' posts found.');
}
return allPosts;
// this is passed into getPostContent() for the markdown conversion
const turndownService = translator.initTurndownService();

let allPosts = [];
postTypes.forEach((postType) => {
const postsForType = getItemsOfType(data, postType)
.filter(
(post) => post.status[0] !== "trash" && post.status[0] !== "draft"
)
.map((post) => ({
// meta data isn't written to file, but is used to help with other things
meta: {
id: getPostId(post),
slug: getPostSlug(post),
coverImageId: getPostCoverImageId(post),
type: postType,
imageUrls: [],
},
frontmatter: {
title: getPostTitle(post),
slug: getPostSlug(post),
description: getSeoDescription(post),
date: getPostDate(post),
keywords: getSeoKeywords(post),
categories: getCategories(post),
tags: getTags(post),
datePublishedGmt: getDateGmt(post),
dateModifiedGmt: getModifiedGmt(post),
originLink: post.link[0],
postId: getPostId(post),
postParent: getPostParent(post),
},
content: translator.getPostContent(post, turndownService, config),
}));

if (postTypes.length > 1) {
console.log(`${postsForType.length} "${postType}" posts found.`);
}

allPosts.push(...postsForType);
});

if (postTypes.length === 1) {
console.log(allPosts.length + " posts found.");
}
return allPosts;
}

function getPostId(post) {
return post.post_id[0];
return post.post_id[0];
}
function getPostParent(post) {
return post.post_parent[0];
}

function getPostSlug(post) {
return decodeURIComponent(post.post_name[0]);
return decodeURIComponent(post.post_name[0]);
}

function getPostCoverImageId(post) {
if (post.postmeta === undefined) {
return undefined;
}

const postmeta = post.postmeta.find(postmeta => postmeta.meta_key[0] === '_thumbnail_id');
const id = postmeta ? postmeta.meta_value[0] : undefined;
return id;
if (post.postmeta === undefined) {
return undefined;
}

const postmeta = post.postmeta.find(
(postmeta) => postmeta.meta_key[0] === "_thumbnail_id"
);
const id = postmeta ? postmeta.meta_value[0] : undefined;
return id;
}
// get yoast seo meta tags
function getSeoTitle(post) {
if (post.postmeta === undefined) {
return undefined;
}

const postmeta = post.postmeta.find(
(postmeta) => postmeta.meta_key[0] === "_yoast_wpseo_title"
);
const id = postmeta ? postmeta.meta_value[0] : undefined;
return id;
}
function getSeoDescription(post) {
if (post.postmeta === undefined) {
return undefined;
}

const postmeta = post.postmeta.find(
(postmeta) => postmeta.meta_key[0] === "_yoast_wpseo_metadesc"
);
const id = postmeta ? postmeta.meta_value[0] : undefined;
return id;
}
function getSeoKeywords(post) {
if (post.postmeta === undefined) {
return undefined;
}

const postmeta = post.postmeta.find(
(postmeta) => postmeta.meta_key[0] === "_yoast_wpseo_focuskw"
);
const id = postmeta ? postmeta.meta_value[0] : undefined;
return id;
}

function getPostTitle(post) {
return post.title[0];
return titleCase(post.title[0]);
}
// function getFrontmatterSlug(post) {
// const url = new URL(post.link[0]);
// return decodeURIComponent(url.pathname.replace(/\//g, ''));
// }
function getPostExcerpt(post) {
const description = post.encoded[1].replace(/(\r\n|\n|\r)/gm, " ");
return description;
}

function getPostDate(post) {
const dateTime = luxon.DateTime.fromRFC2822(post.pubDate[0], { zone: 'utc' });

if (settings.custom_date_formatting) {
return dateTime.toFormat(settings.custom_date_formatting);
} else if (settings.include_time_with_date) {
return dateTime.toISO();
} else {
return dateTime.toISODate();
}
const dateTime = luxon.DateTime.fromRFC2822(post.pubDate[0], {
zone: "utc+8",
});

if (settings.custom_date_formatting) {
return dateTime.toFormat(settings.custom_date_formatting);
} else if (settings.include_time_with_date) {
return dateTime.toISO();
} else {
return dateTime.toISODate();
}
}

function getDateGmt(post) {
return post.post_date_gmt[0];
}
function getModifiedGmt(post) {
if (post.post_modified_gmt === undefined) {
return undefined;
}
return post.post_modified_gmt[0];
}
function getCategories(post) {
const categories = processCategoryTags(post, 'category');
return categories.filter(category => !settings.filter_categories.includes(category));
const categories = processCategoryTags(post, "category");
return categories.filter(
(category) => !settings.filter_categories.includes(category)
);
}

function getTags(post) {
return processCategoryTags(post, 'post_tag');
return processCategoryTags(post, "post_tag");
}

function processCategoryTags(post, domain) {
if (!post.category) {
return [];
}

return post.category
.filter(category => category.$.domain === domain)
.map(({ $: attributes }) => decodeURIComponent(attributes.nicename));
if (!post.category) {
return [];
}

return (
post.category
.filter((category) => category.$.domain === domain)
// .map(({ $: attributes }) => decodeURIComponent(attributes.nicename));
.map(({ $: attributes }) =>
titleCase(attributes.nicename).replace(/-/g, " ")
)
);
}

function collectAttachedImages(data) {
Expand Down
Loading