{ "name": "HasData - Page Markdown changes to Slack (test candidate)", "flow": [ { "id": 1, "module": "util:SetVariables", "version": 1, "parameters": {}, "mapper": { "variables": [ { "name": "url", "value": "https://docs.hasdata.com/apis/web-scraping-api/features/output-formats" } ], "scope": "roundtrip" }, "metadata": { "designer": { "x": 0, "y": 0 }, "interface": [ { "name": "url", "type": "text", "label": "url" } ] } }, { "id": 2, "module": "hasdata:webScraping", "version": 1, "parameters": {}, "mapper": { "url": "{{1.url}}", "proxyType": "datacenter", "proxyCountry": "US", "blockResources": true, "blockAds": true, "screenshot": false, "jsRendering": true, "extractEmails": false, "extractLinks": false, "outputFormat": [ "json", "markdown" ] }, "metadata": { "designer": { "x": 300, "y": 0 }, "expect": [ { "name": "url", "type": "url", "label": "URL", "required": true }, { "name": "headers", "type": "json", "label": "Headers" }, { "name": "proxyType", "type": "select", "label": "Proxy Type", "validate": { "enum": [ "datacenter", "residential" ] } }, { "name": "proxyCountry", "type": "select", "label": "Proxy Country", "validate": { "enum": [ "US", "UK", "DE", "IE", "FR", "IT", "SE", "BR", "CA", "JP", "SG", "IN", "ID" ] } }, { "name": "blockResources", "type": "boolean", "label": "Block Images And CSS" }, { "name": "blockAds", "type": "boolean", "label": "Block ADS" }, { "name": "blockUrls", "type": "json", "label": "Block URLs" }, { "name": "wait", "type": "integer", "label": "Wait For A Time" }, { "name": "waitFor", "type": "text", "label": "Wait For CSS Selector" }, { "name": "jsScenario", "type": "json", "label": "JavaScript Execution" }, { "name": "extractRules", "type": "json", "label": "Extraction Rules" }, { "name": "screenshot", "type": "boolean", "label": "Screenshot" }, { "name": "jsRendering", "type": "boolean", "label": "JS Rendering" }, { "name": "extractEmails", "type": "boolean", "label": "Extract Emails" }, { "name": "extractLinks", "type": "boolean", "label": "Extract Links" }, { "name": "includeOnlyTags", "type": "json", "label": "Include Only Tags" }, { "name": "excludeTags", "type": "json", "label": "Exclude Tags" }, { "name": "removeBase64Images", "type": "boolean", "label": "Remove Base64 Images" }, { "name": "outputFormat", "type": "select", "label": "Output Format", "multiple": true, "validate": { "enum": [ "json", "html", "markdown", "text" ] } }, { "name": "aiExtractRules", "type": "json", "label": "AI Extraction Rules" } ], "parameters": [ { "name": "__IMTCONN__", "type": "account:hasdata", "label": "Connection", "required": true } ] } }, { "id": 3, "module": "util:SetVariables", "version": 1, "parameters": {}, "mapper": { "variables": [ { "name": "contentHash", "value": "{{sha256(2.markdown)}}" }, { "name": "checkedAt", "value": "{{formatDate(now; \"YYYY-MM-DD HH:mm:ss\"; \"UTC\")}}" } ], "scope": "roundtrip" }, "metadata": { "designer": { "x": 600, "y": 0 }, "interface": [ { "name": "contentHash", "type": "text", "label": "contentHash" }, { "name": "checkedAt", "type": "text", "label": "checkedAt" } ] }, "filter": { "name": "Successful nonempty Markdown only", "conditions": [ [ { "a": "{{2.requestMetadata.status}}", "b": "ok", "o": "text:equal" }, { "a": "{{2.statusCode}}", "b": "199", "o": "number:greater" }, { "a": "{{2.statusCode}}", "b": "300", "o": "number:less" }, { "a": "{{2.markdown}}", "o": "exist" }, { "a": "{{length(trim(2.markdown))}}", "b": "0", "o": "number:greater" } ] ] } }, { "id": 4, "module": "datastore:ExistRecord", "version": 1, "parameters": {}, "mapper": { "key": "page-v1|{{sha256(1.url)}}" }, "metadata": { "designer": { "x": 900, "y": 0 }, "parameters": [ { "name": "datastore", "type": "datastore", "label": "Data store", "required": true } ], "expect": [ { "name": "key", "type": "text", "label": "Key", "required": true } ] } }, { "id": 5, "module": "builtin:BasicRouter", "version": 1, "parameters": {}, "mapper": null, "metadata": { "designer": { "x": 1200, "y": 0 } }, "routes": [ { "flow": [ { "id": 6, "module": "datastore:AddRecord", "version": 1, "parameters": {}, "mapper": { "key": "page-v1|{{sha256(1.url)}}", "overwrite": false, "data": { "url": "{{1.url}}", "contentHash": "{{3.contentHash}}", "checkedAt": "{{3.checkedAt}}" } }, "metadata": { "designer": { "x": 1500, "y": -150 }, "parameters": [ { "name": "datastore", "type": "datastore", "label": "Data store", "required": true } ], "expect": [ { "name": "key", "type": "text", "label": "Key" }, { "name": "overwrite", "type": "boolean", "label": "Overwrite an existing record", "required": true }, { "name": "data", "spec": [ { "name": "url", "type": "text", "label": "URL" }, { "name": "contentHash", "type": "text", "label": "Content SHA-256" }, { "name": "checkedAt", "type": "text", "label": "Accepted snapshot at (UTC)" } ], "type": "collection", "label": "Record" } ] }, "filter": { "name": "First observation: save baseline", "conditions": [ [ { "a": "{{4.exist}}", "b": "false", "o": "text:equal" } ] ] } } ] }, { "flow": [ { "id": 7, "module": "datastore:GetRecord", "version": 1, "parameters": {}, "mapper": { "key": "page-v1|{{sha256(1.url)}}", "returnWrapped": false }, "metadata": { "designer": { "x": 1500, "y": 150 }, "parameters": [ { "name": "datastore", "type": "datastore", "label": "Data store", "required": true } ], "expect": [ { "name": "key", "type": "text", "label": "Key", "required": true }, { "name": "returnWrapped", "type": "boolean", "label": "Return Wrapped Output", "required": true } ], "interface": [ { "name": "url", "type": "text", "label": "URL" }, { "name": "contentHash", "type": "text", "label": "Content SHA-256" }, { "name": "checkedAt", "type": "text", "label": "Accepted snapshot at (UTC)" } ] }, "filter": { "name": "A baseline already exists", "conditions": [ [ { "a": "{{4.exist}}", "b": "true", "o": "text:equal" } ] ] } }, { "id": 8, "module": "slack:CreateMessage", "version": 4, "parameters": {}, "mapper": { "channelWType": "list", "channelType": "private", "channel": "", "text": "Page content changed\nURL: {{escapeHTML(1.url)}}\nPrevious accepted snapshot (UTC): {{7.checkedAt}}\nCurrent check (UTC): {{3.checkedAt}}\nCurrent excerpt (not a diff):\n{{escapeHTML(substring(2.markdown; 0; 1500))}}", "parse": false, "mrkdwn": false, "link_names": false, "unfurl_links": false, "unfurl_media": false }, "metadata": { "designer": { "x": 1800, "y": 150 }, "parameters": [ { "name": "__IMTCONN__", "type": "account:slack2,slack3", "label": "Connection", "required": true } ], "expect": [ { "name": "channelWType", "type": "select", "label": "Enter a channel ID or name", "required": true, "validate": { "enum": [ "manualy", "list" ] } }, { "name": "text", "type": "text", "label": "Text" }, { "name": "blocks", "type": "text", "label": "Blocks" }, { "name": "thread_ts", "type": "text", "label": "Thread message ID (time stamp)" }, { "name": "reply_broadcast", "type": "boolean", "label": "Reply broadcast" }, { "name": "link_names", "type": "boolean", "label": "Link names" }, { "name": "parse", "type": "boolean", "label": "Parse message text" }, { "name": "mrkdwn", "type": "boolean", "label": "Use markdown" }, { "name": "unfurl_links", "type": "boolean", "label": "Unfurl primarily text-based content" }, { "name": "unfurl_media", "type": "boolean", "label": "Unfurl media content" }, { "name": "icon_emoji", "type": "text", "label": "Icon emoji" }, { "name": "icon_url", "type": "url", "label": "Icon url" }, { "name": "username", "type": "text", "label": "User name" }, { "name": "channelType", "type": "select", "label": "Channel type", "required": true, "validate": { "enum": [ "public", "private", "im", "mpim" ] } }, { "name": "channel", "type": "select", "label": "Private channel", "required": true } ] }, "filter": { "name": "Stored and current content differ", "conditions": [ [ { "a": "{{7.contentHash}}", "o": "exist" }, { "a": "{{7.contentHash}}", "b": "{{3.contentHash}}", "o": "text:notequal" } ] ] } }, { "id": 9, "module": "datastore:AddRecord", "version": 1, "parameters": {}, "mapper": { "key": "page-v1|{{sha256(1.url)}}", "overwrite": true, "data": { "url": "{{1.url}}", "contentHash": "{{3.contentHash}}", "checkedAt": "{{3.checkedAt}}" } }, "metadata": { "designer": { "x": 2100, "y": 150 }, "parameters": [ { "name": "datastore", "type": "datastore", "label": "Data store", "required": true } ], "expect": [ { "name": "key", "type": "text", "label": "Key" }, { "name": "overwrite", "type": "boolean", "label": "Overwrite an existing record", "required": true }, { "name": "data", "spec": [ { "name": "url", "type": "text", "label": "URL" }, { "name": "contentHash", "type": "text", "label": "Content SHA-256" }, { "name": "checkedAt", "type": "text", "label": "Accepted snapshot at (UTC)" } ], "type": "collection", "label": "Record" } ] } } ] } ] } ], "metadata": { "instant": false, "version": 1, "scenario": { "roundtrips": 1, "maxErrors": 3, "autoCommit": true, "autoCommitTriggerLast": true, "sequential": true, "slots": null, "confidential": false, "dataloss": false, "dlq": false, "freshVariables": false }, "designer": { "orphans": [] }, "notes": [] } }