Added Dockerfile and CI/CD for Imapsync
Added Core Functionality for Imapsync
This commit is contained in:
414
tools/imapsync/index.mjs
Normal file
414
tools/imapsync/index.mjs
Normal file
@@ -0,0 +1,414 @@
|
||||
import Imap from "node-imap"
|
||||
|
||||
import {inspect} from "util"
|
||||
import {createClient} from "@supabase/supabase-js";
|
||||
|
||||
|
||||
let key = process.env.SUPABASE_KEY
|
||||
let url = process.env.SUPABASE_URL
|
||||
let interval = process.env.INTERVAL
|
||||
|
||||
|
||||
|
||||
|
||||
const supabase = createClient(url,key)
|
||||
|
||||
import {simpleParser} from "mailparser"
|
||||
|
||||
let tenants = (await supabase.from("tenants").select()).data
|
||||
console.log(tenants)
|
||||
|
||||
//Function Declarations
|
||||
|
||||
function openInbox(imapClient, cb) {
|
||||
imapClient.openBox('INBOX', false, cb);
|
||||
}
|
||||
|
||||
const uploadAttachmentToSupabase = async (tenant, folder ,file ) => {
|
||||
console.log("Start Upload")
|
||||
console.log(tenant)
|
||||
console.log(folder)
|
||||
console.log(file)
|
||||
|
||||
//Create Blob from File Content with Mime Type
|
||||
let blob = new Blob([file.content])
|
||||
blob = blob.slice(0,blob.size, file.contentType)
|
||||
|
||||
|
||||
|
||||
const {data: storageData,error: storageError} = await supabase.storage
|
||||
.from("documents")
|
||||
.upload(`${tenant}/${folder}/${file.filename}`,blob)
|
||||
|
||||
console.log(storageData)
|
||||
console.log(storageError)
|
||||
|
||||
if(storageData) {
|
||||
const {data: tableData,error: tableError} = await supabase
|
||||
.from("documents")
|
||||
.insert({
|
||||
object: storageData.id,
|
||||
folder: folder,
|
||||
tags: ["E-Mail Anhang"],
|
||||
path: storageData.path
|
||||
|
||||
})
|
||||
.select()
|
||||
|
||||
console.log(tableData)
|
||||
console.log(tableError)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
function processEmails () {
|
||||
if(!key || !url ||!interval) {
|
||||
console.log("Missing atleast one Config Var")
|
||||
|
||||
return
|
||||
}
|
||||
tenants.forEach(tenant => {
|
||||
if(tenant.email_data) {
|
||||
if(tenant.email_data.imap) {
|
||||
let imapData = tenant.email_data.imap
|
||||
let imap = new Imap({
|
||||
user: imapData.user,
|
||||
password: imapData.password,
|
||||
host: imapData.host, //this may differ if you are using some other mail services like yahoo
|
||||
port: imapData.port,
|
||||
tls: imapData.tls,
|
||||
connTimeout: 10000, // Default by node-imap
|
||||
authTimeout: 5000, // Default by node-imap,
|
||||
debug: null, // Or your custom function with only one incoming argument. Default: null
|
||||
tlsOptions: { rejectUnauthorized: false },
|
||||
mailbox: imapData.mailbox, // mailbox to monitor
|
||||
searchFilter: ["UNSEEN", "FLAGGED"], // the search filter being used after an IDLE notification has been retrieved
|
||||
markSeen: true, // all fetched email willbe marked as seen and not fetched next time
|
||||
fetchUnreadOnStart: true, // use it only if you want to get all unread email on lib start. Default is `false`,
|
||||
mailParserOptions: { streamAttachments: true }, // options to be passed to mailParser lib.
|
||||
attachments: true, // download attachments as they are encountered to the project directory
|
||||
attachmentOptions: { directory: "attachments/" } // specify a download directory for attachments
|
||||
})
|
||||
|
||||
let tenantEmailArray = []
|
||||
|
||||
imap.once('ready', function() {
|
||||
console.log("start open inbox for Tenant: " + tenant.name);
|
||||
try{
|
||||
openInbox(imap, function (err, box) {
|
||||
imap.search([ 'UNSEEN', ['SINCE', "1 January 2023"] ], function(err, results) {
|
||||
if(!results || !results.length){
|
||||
console.log("The server didn't find any emails matching the specified criteria")
|
||||
imap.end();return;
|
||||
}
|
||||
|
||||
var f = imap.fetch(results,{ //you can set amount range like '1:2' or 'results' for all results
|
||||
bodies: '',
|
||||
struct: true
|
||||
})
|
||||
|
||||
f.on('message', function(msg, seqno) {
|
||||
console.log('Message #%d', seqno);
|
||||
var prefix = '(#' + seqno + ') ';
|
||||
msg.on('body', function(stream, info) {
|
||||
//Retrieve the 'from' header and buffer the entire body of the newest message:
|
||||
if (info.which === 'TEXT')
|
||||
|
||||
var buffer = '', count = 0;
|
||||
|
||||
stream.on('data', async function(chunk) {
|
||||
count += chunk.length;
|
||||
buffer += chunk.toString('utf8');
|
||||
});
|
||||
|
||||
stream.once('end', async function() {
|
||||
let attach = null
|
||||
//console.log((await simpleParser(buffer))) -> to see entire data of email
|
||||
let attachments = (await simpleParser(buffer)).attachments
|
||||
/* if(((await simpleParser(buffer)).attachments).length != 0) {
|
||||
|
||||
|
||||
attach = (await simpleParser(buffer)).attachments[0].content //to get attachments
|
||||
let file = (await simpleParser(buffer)).attachments[0]
|
||||
console.log(file)
|
||||
fs.writeFile(file.filename,file.content, "binary", (err) => {
|
||||
if(!err) console.log("Success")
|
||||
})
|
||||
}*/
|
||||
|
||||
if (info.which !== 'TEXT'){
|
||||
let dataheader = Imap.parseHeader(buffer)
|
||||
|
||||
//start -> set data, that you want to save on your DB
|
||||
let emails_data = {
|
||||
"date": dataheader.date[0],
|
||||
"subject": dataheader.subject[0],
|
||||
"from": dataheader.from[0],
|
||||
"to": dataheader.to[0],
|
||||
"content": (await simpleParser(buffer)).text,
|
||||
"attachments": attachments
|
||||
}
|
||||
//console.log(emails_data)
|
||||
//end -> set data
|
||||
|
||||
attachments.forEach(attachment => uploadAttachmentToSupabase(tenant.id, "Eingang", attachment))
|
||||
|
||||
|
||||
|
||||
//tenantEmailArray.push(emails_data)
|
||||
}
|
||||
else
|
||||
console.log(prefix + 'Body [%s] Finished', inspect(info.which));
|
||||
});
|
||||
});
|
||||
|
||||
//mark attributes email as read
|
||||
msg.once('attributes', function(attrs) {
|
||||
let uid = attrs.uid;
|
||||
imap.addFlags(uid, ['\\Seen'], function (err) {
|
||||
if (err) {
|
||||
console.log(err);
|
||||
} else {
|
||||
console.log("Done, marked email as read!")
|
||||
}
|
||||
});
|
||||
});
|
||||
msg.once('end', function() {
|
||||
console.log(prefix + 'Finished');
|
||||
});
|
||||
});
|
||||
|
||||
f.once('error', function(err) {
|
||||
console.log('Fetch error: ' + err);
|
||||
});
|
||||
|
||||
f.once('end', function() {
|
||||
console.log('Done fetching all messages!');
|
||||
imap.end();
|
||||
});
|
||||
|
||||
})
|
||||
});} catch (err){
|
||||
console.log("Error when request open inbox mail",err)
|
||||
}
|
||||
});
|
||||
|
||||
imap.once('error', function(err) {
|
||||
console.log("Error when connection to IMAP from Tenant: " + tenant.name, err);
|
||||
});
|
||||
|
||||
imap.once('close', function() {
|
||||
console.log('Connection ended');
|
||||
});
|
||||
|
||||
imap.connect();
|
||||
|
||||
imap.once('close', async () => {
|
||||
//console.log(email_array);
|
||||
console.log("Read Finished")
|
||||
//console.log(tenantEmailArray)
|
||||
/*tenantEmailArray.forEach(email => {
|
||||
console.log("PENIS")
|
||||
//console.log(email)
|
||||
console.log(email.attachments)
|
||||
if(email.attachments.length > 0) {
|
||||
console.log("Attachments there")
|
||||
email.attachments.forEach(attachment => uploadAttachmentToSupabase(tenant.id, "Eingang", attachment) )
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
})*/
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
setInterval(processEmails,interval * 1000)
|
||||
|
||||
|
||||
|
||||
/*
|
||||
|
||||
let imap = new Imap({
|
||||
user: "info@federspiel.tech",
|
||||
password: "fpGUpbQG34",
|
||||
host: "imap.strato.de", //this may differ if you are using some other mail services like yahoo
|
||||
port: 993,
|
||||
tls: true,
|
||||
connTimeout: 10000, // Default by node-imap
|
||||
authTimeout: 5000, // Default by node-imap,
|
||||
debug: null, // Or your custom function with only one incoming argument. Default: null
|
||||
tlsOptions: { rejectUnauthorized: false },
|
||||
mailbox: "INBOX", // mailbox to monitor
|
||||
searchFilter: ["UNSEEN", "FLAGGED"], // the search filter being used after an IDLE notification has been retrieved
|
||||
markSeen: true, // all fetched email willbe marked as seen and not fetched next time
|
||||
fetchUnreadOnStart: true, // use it only if you want to get all unread email on lib start. Default is `false`,
|
||||
mailParserOptions: { streamAttachments: true }, // options to be passed to mailParser lib.
|
||||
attachments: true, // download attachments as they are encountered to the project directory
|
||||
attachmentOptions: { directory: "attachments/" } // specify a download directory for attachments
|
||||
});
|
||||
|
||||
|
||||
let email_array = [];
|
||||
|
||||
imap.once('ready', function() {
|
||||
console.log("start open inbox");
|
||||
try{
|
||||
openInbox(function (err, box) {
|
||||
imap.search([ 'UNSEEN', ['SINCE', "1 January 2023"] ], function(err, results) {
|
||||
if(!results || !results.length){
|
||||
console.log("The server didn't find any emails matching the specified criteria")
|
||||
imap.end();return;
|
||||
}
|
||||
|
||||
var f = imap.fetch(results,{ //you can set amount range like '1:2' or 'results' for all results
|
||||
bodies: '',
|
||||
struct: true
|
||||
})
|
||||
|
||||
f.on('message', function(msg, seqno) {
|
||||
console.log('Message #%d', seqno);
|
||||
var prefix = '(#' + seqno + ') ';
|
||||
msg.on('body', function(stream, info) {
|
||||
//Retrieve the 'from' header and buffer the entire body of the newest message:
|
||||
if (info.which === 'TEXT')
|
||||
|
||||
var buffer = '', count = 0;
|
||||
|
||||
stream.on('data', async function(chunk) {
|
||||
count += chunk.length;
|
||||
buffer += chunk.toString('utf8');
|
||||
});
|
||||
|
||||
stream.once('end', async function() {
|
||||
let attach = null
|
||||
//console.log((await simpleParser(buffer))) -> to see entire data of email
|
||||
let attachments = (await simpleParser(buffer)).attachments
|
||||
/!* if(((await simpleParser(buffer)).attachments).length != 0) {
|
||||
|
||||
|
||||
attach = (await simpleParser(buffer)).attachments[0].content //to get attachments
|
||||
let file = (await simpleParser(buffer)).attachments[0]
|
||||
console.log(file)
|
||||
fs.writeFile(file.filename,file.content, "binary", (err) => {
|
||||
if(!err) console.log("Success")
|
||||
})
|
||||
}*!/
|
||||
|
||||
if (info.which !== 'TEXT'){
|
||||
let dataheader = Imap.parseHeader(buffer)
|
||||
|
||||
//start -> set data, that you want to save on your DB
|
||||
let emails_data = {
|
||||
"date": dataheader.date[0],
|
||||
"subject": dataheader.subject[0],
|
||||
"from": dataheader.from[0],
|
||||
"to": dataheader.to[0],
|
||||
"content": (await simpleParser(buffer)).text,
|
||||
"attachments": attachments
|
||||
}
|
||||
console.log(emails_data)
|
||||
//end -> set data
|
||||
|
||||
email_array.push(emails_data)
|
||||
}
|
||||
else
|
||||
console.log(prefix + 'Body [%s] Finished', inspect(info.which));
|
||||
});
|
||||
});
|
||||
|
||||
//mark attributes email as read
|
||||
msg.once('attributes', function(attrs) {
|
||||
let uid = attrs.uid;
|
||||
/!*imap.addFlags(uid, ['\\Seen'], function (err) {
|
||||
if (err) {
|
||||
console.log(err);
|
||||
} else {
|
||||
console.log("Done, marked email as read!")
|
||||
}
|
||||
});*!/
|
||||
});
|
||||
msg.once('end', function() {
|
||||
console.log(prefix + 'Finished');
|
||||
});
|
||||
});
|
||||
|
||||
f.once('error', function(err) {
|
||||
console.log('Fetch error: ' + err);
|
||||
});
|
||||
|
||||
f.once('end', function() {
|
||||
console.log('Done fetching all messages!');
|
||||
imap.end();
|
||||
});
|
||||
|
||||
})
|
||||
});} catch (err){
|
||||
console.log("Error when request open inbox mail",err)
|
||||
}
|
||||
});*/
|
||||
/*
|
||||
imap.once('error', function(err) {
|
||||
console.log("Error when connection to IMAP", err);
|
||||
});
|
||||
|
||||
imap.once('close', function() {
|
||||
console.log('Connection ended');
|
||||
});
|
||||
|
||||
imap.connect();*/
|
||||
|
||||
|
||||
/*
|
||||
imap.once('close', async () => {
|
||||
//console.log(email_array);
|
||||
email_array.forEach(email => {
|
||||
console.log(email)
|
||||
|
||||
|
||||
|
||||
})
|
||||
|
||||
|
||||
});*/
|
||||
/*
|
||||
let buffer = fs.readFileSync("Rechnung40980272.pdf")
|
||||
let blob = new Blob([buffer])
|
||||
blob = blob.slice(0,blob.size, "application/pdf")
|
||||
|
||||
console.log(blob)
|
||||
|
||||
const {dataS,error} = await supabase
|
||||
.storage
|
||||
.from("documents")
|
||||
.upload("test/test.pdf", blob)
|
||||
|
||||
console.log(dataS)
|
||||
console.log(error)*/
|
||||
|
||||
/*
|
||||
fs.readFile("Rechnung40980272.pdf",'utf-8', async (err, data) => {
|
||||
console.log(data)
|
||||
const blob = new Blob([data])
|
||||
console.log(blob)
|
||||
console.log(mime.getType(data))
|
||||
|
||||
/!*const {dataS,error} = await supabase
|
||||
.storage
|
||||
.from("documents")
|
||||
.upload("test/test.pdf", data)*!/
|
||||
|
||||
/!*
|
||||
console.log(dataS)
|
||||
console.log(error)*!/
|
||||
})
|
||||
*/
|
||||
Reference in New Issue
Block a user