Products Feed: Convert EditorJS product descriptions to plain text (#489)

* Cache query cursors for the product feed

* Fix missing first page of products

* Add S3 upload

* Explain sze limit on multipart upload

* Change the name of function

* Update the dependencies

* Revert api response size override

* Fix multi part upload

* Remove duplicated code

* Add channel name to the file URL

* Render EditorJS formatted descriptions as plaintext.
SEO Description field will be removed

* Add changeset

* Improve tests and allow escaped signs

.changeset/fast-adults-deny.md

@@ -0,0 +1,5 @@
+---
+"saleor-app-products-feed": patch
+---
+
+Product description in the feed is now a plaintext instead of JSON.

apps/products-feed/src/lib/editor-js-plaintext-renderer.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from "vitest";
+import { EditorJsPlaintextRenderer } from "./editor-js-plaintext-renderer";
+
+describe("EditorJsPlaintextRenderer", () => {
+  it("Empty response for invalid input", () => {
+    expect(EditorJsPlaintextRenderer({ stringData: "not json" })).toBe(undefined);
+    expect(EditorJsPlaintextRenderer({ stringData: "" })).toBe(undefined);
+  });
+  it("Returns plaintext with no formatting when passed paragraph block", () => {
+    expect(
+      EditorJsPlaintextRenderer({
+        stringData:
+          '{"time": 1684697732024, "blocks": [{"id": "HVJ8gMNIXY", "data": {"text": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris nibh lacus, dignissim at aliquet et, gravida sed velit. Suspendisse at volutpat erat. Lorem ipsum dolor sit amet, consectetur adipiscing elit."}, "type": "paragraph"}], "version": "2.24.3"}',
+      })
+    ).toBe(
+      "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris nibh lacus, dignissim at aliquet et, gravida sed velit. Suspendisse at volutpat erat. Lorem ipsum dolor sit amet, consectetur adipiscing elit."
+    );
+  });
+  it("Returns plaintext with no formatting when passed paragraph block with additional styles", () => {
+    expect(
+      EditorJsPlaintextRenderer({
+        stringData:
+          '{"time": 1684697809104, "blocks": [{"id": "HVJ8gMNIXY", "data": {"text": "Lorem ipsum dolor sit <b>amet</b>, consectetur adipiscing elit. Mauris <s>nibh lacus</s>, dignissim at aliquet et, gravida sed velit. Suspendisse at volutpat erat. <i>Lorem ipsum </i>dolor sit amet, consectetur adipiscing elit."}, "type": "paragraph"}], "version": "2.24.3"}',
+      })
+    ).toBe(
+      "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Mauris nibh lacus, dignissim at aliquet et, gravida sed velit. Suspendisse at volutpat erat. Lorem ipsum dolor sit amet, consectetur adipiscing elit."
+    );
+  });
+  it("Returns text containing angle brackets, when passed block without the style tags", () => {
+    expect(
+      EditorJsPlaintextRenderer({
+        stringData:
+          '{"time": 1684748620371, "blocks": [{"id": "fw-PCw9s-0", "data": {"text": "Everybody knows that 1 &lt; 2 and 1 &gt; 0."}, "type": "paragraph"}, {"id": "eUK1ih8Wmz", "data": {"text": "This is text heart: &lt;3"}, "type": "paragraph"}], "version": "2.24.3"}',
+      })
+    ).toBe("Everybody knows that 1 &lt; 2 and 1 &gt; 0.\nThis is text heart: &lt;3");
+    it("Returns numbered list when passed ordered list block", () => {
+      expect(
+        EditorJsPlaintextRenderer({
+          stringData:
+            '{"time": 1684697916091, "blocks": [{"id": "BNL219JhYr", "data": {"items": ["Apples", "Oranges", "Bananas"], "style": "ordered"}, "type": "list"}], "version": "2.24.3"}',
+        })
+      ).toBe("1. Apples\n2. Oranges\n3. Bananas");
+    });
+    it("Returns list with dashes when passed unordered list block", () => {
+      expect(
+        EditorJsPlaintextRenderer({
+          stringData:
+            '{"time": 1684697984679, "blocks": [{"id": "BNL219JhYr", "data": {"items": ["Apples", "Oranges", "Bananas"], "style": "unordered"}, "type": "list"}], "version": "2.24.3"}',
+        })
+      ).toBe("- Apples\n- Oranges\n- Bananas");
+    });
+    it("Returns plaintext when header block is passed", () => {
+      expect(
+        EditorJsPlaintextRenderer({
+          stringData:
+            '{"time": 1684698075115, "blocks": [{"id": "nC-oNRu-pp", "data": {"text": "Lorem ipsum", "level": 1}, "type": "header"}], "version": "2.24.3"}',
+        })
+      ).toBe("Lorem ipsum");
+    });
+    it("Returns text additional new line after header, when theres another block passed", () => {
+      expect(
+        EditorJsPlaintextRenderer({
+          stringData:
+            '{"time": 1684748016130, "blocks": [{"id": "nC-oNRu-pp", "data": {"text": "This is header", "level": 1}, "type": "header"}, {"id": "fw-PCw9s-0", "data": {"text": "There should be additional new line between header and paragraph"}, "type": "paragraph"}], "version": "2.24.3"}',
+        })
+      ).toBe("This is header\n\nThere should be additional new line between header and paragraph");
+    });
+    it("Returns text when passed all types of blocks", () => {
+      expect(
+        EditorJsPlaintextRenderer({
+          stringData:
+            '{"time": 1684698250098, "blocks": [{"id": "nC-oNRu-pp", "data": {"text": "Lorem ipsum", "level": 1}, "type": "header"}, {"id": "1ADVi9cvw8", "data": {"text": "This is <b>introduction</b> to the list of things"}, "type": "paragraph"}, {"id": "7OFi_vE_hc", "data": {"items": ["Red", "Blue"], "style": "ordered"}, "type": "list"}, {"id": "PYLABJ1KWZ", "data": {"text": "Closing thoughts."}, "type": "paragraph"}], "version": "2.24.3"}',
+        })
+      ).toBe(
+        "Lorem ipsum\n\nThis is introduction to the list of things\n1. Red\n2. Blue\nClosing thoughts."
+      );
+    });
+  });
+});

apps/products-feed/src/lib/editor-js-plaintext-renderer.ts

@@ -0,0 +1,87 @@
+interface ParagraphData {
+  text: string;
+}
+
+interface HeaderData {
+  text: string;
+  level: number;
+}
+
+interface ListData {
+  items: string[];
+  style: "unordered" | "ordered";
+}
+
+interface esjBlock {
+  id: string;
+  type: string;
+  data: ParagraphData | HeaderData | ListData | Record<never, never>;
+}
+
+interface ejsData {
+  version: string;
+  time: number;
+  blocks: esjBlock[];
+}
+
+const renderParagraph = (data: ParagraphData) => {
+  return data.text;
+};
+
+const renderHeader = (data: HeaderData) => {
+  return data.text + "\n";
+};
+
+const renderList = (data: ListData) => {
+  if (data.style === "ordered") {
+    return data.items.map((item, index) => `${index + 1}. ${item}`).join("\n");
+  }
+  return data.items.map((item) => `- ${item}`).join("\n");
+};
+
+const renderDelimiter = () => {
+  return "\n";
+};
+
+const renderBlock = (block: esjBlock) => {
+  switch (block.type) {
+    case "header":
+      return renderHeader(block.data as HeaderData);
+    case "paragraph":
+      return renderParagraph(block.data as ParagraphData);
+    case "list":
+      return renderList(block.data as ListData);
+    case "delimiter":
+      return renderDelimiter();
+    default:
+      return "";
+  }
+};
+
+const removeHtmlTags = (input: string) => {
+  /*
+   * The EditorJS used in the dashboard produces only a few one letter tags,
+   * like <b> or </s>, so we can use simpler regex to remove them
+   */
+  return input.replace(/<[^>]{1,2}>/g, "");
+};
+
+type EditorJSRendererProps = {
+  stringData: string;
+};
+
+export function EditorJsPlaintextRenderer({ stringData }: EditorJSRendererProps) {
+  let data: ejsData;
+
+  try {
+    data = JSON.parse(stringData) as ejsData;
+  } catch (e) {
+    return;
+  }
+  if (!data) {
+    return;
+  }
+  const { blocks } = data;
+
+  return removeHtmlTags(blocks.map((b) => renderBlock(b)).join("\n")).trim();
+}

apps/products-feed/src/lib/google-feed/generate-google-xml-feed.ts

@@ -2,6 +2,8 @@ import { XMLBuilder } from "fast-xml-parser";
 import { GoogleFeedProductVariantFragment } from "../../../generated/graphql";
 import { productToProxy } from "./product-to-proxy";
 import { shopDetailsToProxy } from "./shop-details-to-proxy";
+import { EditorJsPlaintextRenderer } from "../editor-js-plaintext-renderer";
+
 interface GenerateGoogleXmlFeedArgs {
   productVariants: GoogleFeedProductVariantFragment[];
   storefrontUrl: string;
@@ -36,7 +38,7 @@ export const generateGoogleXmlFeed = ({
       slug: v.product.slug,
       variantId: v.id,
       sku: v.sku || undefined,
-      description: v.product.seoDescription || v.product.description,
+      description: EditorJsPlaintextRenderer({ stringData: v.product.description }),
       availability: v.quantityAvailable && v.quantityAvailable > 0 ? "in_stock" : "out_of_stock",
       category: v.product.category?.name || "unknown",
       googleProductCategory: v.product.category?.googleCategoryId || "",