From 7d635bf3dc3733f11a3f4e26fbeceab313f4c8ea Mon Sep 17 00:00:00 2001 From: TheFlow Date: Tue, 4 Nov 2025 11:13:15 +1300 Subject: [PATCH] feat: Add Phase 2 newsletter enhancements and email templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 Implementation: - Add RSS feed discovery links to footer (Subscribe section) - Create email templates (base-template.html, research-updates-content.html) - Add comprehensive newsletter sending implementation plan - Fix CSP check to exclude email-templates directory Email templates use inline styles for cross-client compatibility (Gmail, Outlook, Apple Mail) and are excluded from CSP checks. Next steps: Install dependencies (handlebars, @sendgrid/mail), implement EmailService, controller methods, and admin UI. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- docs/NEWSLETTER_SENDING_IMPLEMENTATION.md | 554 ++++++++++++++++++ email-templates/README.md | 101 ++++ email-templates/base-template.html | 65 ++ email-templates/research-updates-content.html | 64 ++ public/js/components/footer.js | 9 +- scripts/check-csp-violations.js | 3 + 6 files changed, 795 insertions(+), 1 deletion(-) create mode 100644 docs/NEWSLETTER_SENDING_IMPLEMENTATION.md create mode 100644 email-templates/base-template.html create mode 100644 email-templates/research-updates-content.html diff --git a/docs/NEWSLETTER_SENDING_IMPLEMENTATION.md b/docs/NEWSLETTER_SENDING_IMPLEMENTATION.md new file mode 100644 index 00000000..893b8579 --- /dev/null +++ b/docs/NEWSLETTER_SENDING_IMPLEMENTATION.md @@ -0,0 +1,554 @@ +# Newsletter Sending Implementation Plan + +## Overview + +Implement complete newsletter sending functionality with email service integration, template rendering, and admin controls. + +--- + +## Dependencies to Install + +```bash +npm install --save handlebars +npm install --save @sendgrid/mail +npm install --save html-to-text +``` + +**Why these packages?** +- `handlebars`: Template engine for email rendering (Mustache-compatible) +- `@sendgrid/mail`: SendGrid email service client +- `html-to-text`: Generate plain-text versions from HTML + +--- + +## Environment Variables + +Add to `.env`: + +```bash +# Email Service +SENDGRID_API_KEY=your_key_here +EMAIL_FROM_ADDRESS=research@agenticgovernance.digital +EMAIL_FROM_NAME=Tractatus Research Team + +# Newsletter URLs +NEWSLETTER_UNSUBSCRIBE_BASE_URL=https://agenticgovernance.digital/api/newsletter/unsubscribe +NEWSLETTER_PREFERENCES_BASE_URL=https://agenticgovernance.digital/newsletter/preferences +``` + +**SendGrid Setup:** +1. Create account: https://signup.sendgrid.com/ +2. Verify domain (agenticgovernance.digital) +3. Create API key with "Mail Send" permissions +4. Set `SENDGRID_API_KEY` in `.env` + +--- + +## File Structure + +``` +src/ +├── services/ +│ └── email.service.js # NEW - Email rendering & sending +├── controllers/ +│ └── newsletter.controller.js # MODIFY - Add send() function +├── routes/ +│ └── newsletter.routes.js # MODIFY - Add /send endpoint +└── utils/ + └── template-renderer.util.js # NEW - Handlebars helpers + +email-templates/ +├── base-template.html # EXISTS +├── research-updates-content.html # EXISTS +├── implementation-notes-content.html # TODO +├── governance-discussions-content.html # TODO +└── project-updates-content.html # TODO +``` + +--- + +## Implementation Steps + +### Step 1: Create Email Service (`src/services/email.service.js`) + +```javascript +const sgMail = require('@sendgrid/mail'); +const Handlebars = require('handlebars'); +const { convert: htmlToText } = require('html-to-text'); +const fs = require('fs').promises; +const path = require('path'); +const logger = require('../utils/logger.util'); + +// Initialize SendGrid +if (process.env.SENDGRID_API_KEY) { + sgMail.setApiKey(process.env.SENDGRID_API_KEY); +} else { + logger.warn('[EmailService] SENDGRID_API_KEY not set - email sending disabled'); +} + +class EmailService { + constructor() { + this.templatesPath = path.join(__dirname, '../../email-templates'); + this.templateCache = new Map(); + } + + /** + * Load and compile template + */ + async loadTemplate(templateName) { + if (this.templateCache.has(templateName)) { + return this.templateCache.get(templateName); + } + + const templatePath = path.join(this.templatesPath, `${templateName}.html`); + const templateContent = await fs.readFile(templatePath, 'utf8'); + const compiled = Handlebars.compile(templateContent); + + this.templateCache.set(templateName, compiled); + return compiled; + } + + /** + * Render email from templates + */ + async renderEmail(tier, variables) { + try { + // Load base template + const baseTemplate = await this.loadTemplate('base-template'); + + // Load content module for tier + const contentTemplates = { + 'research': 'research-updates-content', + 'implementation': 'implementation-notes-content', + 'governance': 'governance-discussions-content', + 'project-updates': 'project-updates-content' + }; + + const contentTemplateName = contentTemplates[tier]; + if (!contentTemplateName) { + throw new Error(`Unknown newsletter tier: ${tier}`); + } + + const contentTemplate = await this.loadTemplate(contentTemplateName); + + // Render content module + const renderedContent = contentTemplate(variables); + + // Render final email + const finalHtml = baseTemplate({ + ...variables, + content_body: renderedContent + }); + + // Generate plain text + const plainText = htmlToText(finalHtml, { + wordwrap: 80, + ignoreImages: true, + preserveNewlines: true + }); + + return { html: finalHtml, text: plainText }; + } catch (error) { + logger.error('[EmailService] Template rendering error:', error); + throw error; + } + } + + /** + * Send email via SendGrid + */ + async send({ to, subject, html, text, from = null }) { + if (!process.env.SENDGRID_API_KEY) { + logger.warn('[EmailService] Email sending skipped - no API key'); + return { success: false, error: 'Email service not configured' }; + } + + const fromAddress = from || { + email: process.env.EMAIL_FROM_ADDRESS || 'hello@agenticgovernance.digital', + name: process.env.EMAIL_FROM_NAME || 'Tractatus' + }; + + try { + await sgMail.send({ + to, + from: fromAddress, + subject, + html, + text, + trackingSettings: { + clickTracking: { enable: true, enableText: false }, + openTracking: { enable: true } + } + }); + + logger.info('[EmailService] Email sent successfully', { to, subject }); + return { success: true }; + } catch (error) { + logger.error('[EmailService] Send error:', error); + return { success: false, error: error.message }; + } + } + + /** + * Send newsletter to subscribers + */ + async sendNewsletter({ tier, subject, previewText, variables, testMode = false, testEmail = null }) { + const NewsletterSubscription = require('../models/NewsletterSubscription.model'); + + try { + // Get subscribers for tier + let subscribers; + if (testMode) { + subscribers = [{ email: testEmail, name: 'Test User' }]; + } else { + subscribers = await NewsletterSubscription.findByInterest(tier, { verified: true, active: true }); + } + + if (subscribers.length === 0) { + return { + success: true, + message: 'No subscribers found for tier', + sent: 0, + failed: 0 + }; + } + + // Render base templates + const results = { + sent: 0, + failed: 0, + errors: [] + }; + + // Send to each subscriber + for (const subscriber of subscribers) { + try { + // Generate unsubscribe/preferences links + const unsubscribeLink = `${process.env.NEWSLETTER_UNSUBSCRIBE_BASE_URL}?token=${subscriber.verification_token}`; + const preferencesLink = `${process.env.NEWSLETTER_PREFERENCES_BASE_URL}?token=${subscriber.verification_token}`; + + // Render email with subscriber-specific data + const { html, text } = await this.renderEmail(tier, { + name: subscriber.name || 'there', + tier_name: this.getTierDisplayName(tier), + unsubscribe_link: unsubscribeLink, + preferences_link: preferencesLink, + ...variables + }); + + // Send + const result = await this.send({ + to: subscriber.email, + subject, + html, + text + }); + + if (result.success) { + results.sent++; + } else { + results.failed++; + results.errors.push({ email: subscriber.email, error: result.error }); + } + + } catch (error) { + results.failed++; + results.errors.push({ email: subscriber.email, error: error.message }); + logger.error('[EmailService] Failed to send to subscriber:', { email: subscriber.email, error }); + } + } + + return { + success: true, + message: `Newsletter sent: ${results.sent} successful, ${results.failed} failed`, + sent: results.sent, + failed: results.failed, + errors: results.errors.length > 0 ? results.errors : undefined + }; + + } catch (error) { + logger.error('[EmailService] Newsletter send error:', error); + throw error; + } + } + + getTierDisplayName(tier) { + const names = { + 'research': 'Research Updates', + 'implementation': 'Implementation Notes', + 'governance': 'Governance Discussions', + 'project-updates': 'Project Updates' + }; + return names[tier] || tier; + } +} + +module.exports = new EmailService(); +``` + +### Step 2: Add Method to NewsletterSubscription Model + +Add to `src/models/NewsletterSubscription.model.js`: + +```javascript +/** + * Find subscribers by interest tier + */ +static async findByInterest(interest, options = {}) { + const collection = await getCollection('newsletter_subscriptions'); + + const query = { + interests: interest, + active: options.active !== undefined ? options.active : true + }; + + if (options.verified !== undefined) { + query.verified = options.verified; + } + + return await collection + .find(query) + .sort({ subscribed_at: -1 }) + .limit(options.limit || 10000) + .toArray(); +} +``` + +### Step 3: Add Send Endpoint to Newsletter Controller + +Add to `src/controllers/newsletter.controller.js`: + +```javascript +const emailService = require('../services/email.service'); + +/** + * Send newsletter (admin only) + * POST /api/newsletter/admin/send + */ +exports.send = async (req, res) => { + try { + const { tier, subject, previewText, variables, testMode, testEmail } = req.body; + + // Validation + if (!tier || !subject || !variables) { + return res.status(400).json({ + success: false, + error: 'Missing required fields: tier, subject, variables' + }); + } + + if (testMode && !testEmail) { + return res.status(400).json({ + success: false, + error: 'Test email required when testMode is true' + }); + } + + // Send newsletter + const result = await emailService.sendNewsletter({ + tier, + subject, + previewText, + variables, + testMode: testMode === true, + testEmail + }); + + logger.info('[Newsletter] Newsletter sent', { tier, sent: result.sent, failed: result.failed }); + + res.json(result); + + } catch (error) { + logger.error('[Newsletter] Send error:', error); + res.status(500).json({ + success: false, + error: 'Failed to send newsletter', + details: process.env.NODE_ENV === 'development' ? error.message : undefined + }); + } +}; + +/** + * Preview newsletter (admin only) + * POST /api/newsletter/admin/preview + */ +exports.preview = async (req, res) => { + try { + const { tier, variables } = req.body; + + if (!tier || !variables) { + return res.status(400).json({ + success: false, + error: 'Missing required fields: tier, variables' + }); + } + + const { html } = await emailService.renderEmail(tier, { + name: 'Preview User', + tier_name: emailService.getTierDisplayName(tier), + unsubscribe_link: '#', + preferences_link: '#', + ...variables + }); + + res.send(html); + + } catch (error) { + logger.error('[Newsletter] Preview error:', error); + res.status(500).json({ + success: false, + error: 'Failed to generate preview', + details: process.env.NODE_ENV === 'development' ? error.message : undefined + }); + } +}; +``` + +### Step 4: Add Routes + +Add to `src/routes/newsletter.routes.js`: + +```javascript +// Admin routes +router.post('/admin/send', requireAuth, requireAdmin, csrfProtection, asyncHandler(newsletterController.send)); +router.post('/admin/preview', requireAuth, requireAdmin, csrfProtection, asyncHandler(newsletterController.preview)); +``` + +### Step 5: Update Admin UI JavaScript + +Create `public/js/admin/newsletter-sending.js`: + +```javascript +// Newsletter sending functionality +document.getElementById('send-newsletter-form')?.addEventListener('submit', async (e) => { + e.preventDefault(); + + const tier = document.getElementById('newsletter-tier').value; + const subject = document.getElementById('newsletter-subject').value; + const preview = document.getElementById('newsletter-preview').value; + const contentRaw = document.getElementById('newsletter-content').value; + + if (!tier || !subject || !contentRaw) { + alert('Please fill all required fields'); + return; + } + + let variables; + try { + variables = JSON.parse(contentRaw); + } catch (error) { + alert('Invalid JSON in content field'); + return; + } + + const statusDiv = document.getElementById('send-status'); + statusDiv.className = 'bg-blue-50 border border-blue-200 text-blue-800 px-4 py-3 rounded'; + statusDiv.textContent = 'Sending newsletter...'; + statusDiv.classList.remove('hidden'); + + try { + const csrfToken = document.cookie + .split('; ') + .find(row => row.startsWith('csrf-token=')) + ?.split('=')[1]; + + const response = await fetch('/api/newsletter/admin/send', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-CSRF-Token': csrfToken + }, + body: JSON.stringify({ + tier, + subject, + previewText: preview, + variables, + testMode: false + }) + }); + + const result = await response.json(); + + if (response.ok && result.success) { + statusDiv.className = 'bg-green-50 border border-green-200 text-green-800 px-4 py-3 rounded'; + statusDiv.textContent = `Success! ${result.message}`; + } else { + throw new Error(result.error || 'Send failed'); + } + + } catch (error) { + statusDiv.className = 'bg-red-50 border border-red-200 text-red-800 px-4 py-3 rounded'; + statusDiv.textContent = `Error: ${error.message}`; + } +}); + +// Preview button +document.getElementById('preview-newsletter-btn')?.addEventListener('click', async () => { + // Similar implementation - opens preview in new window +}); + +// Test send button +document.getElementById('test-newsletter-btn')?.addEventListener('click', async () => { + // Similar implementation with testMode: true +}); +``` + +--- + +## Testing Checklist + +- [ ] Install dependencies +- [ ] Set up SendGrid account and API key +- [ ] Configure environment variables +- [ ] Test template rendering (preview endpoint) +- [ ] Test email sending (test mode with your email) +- [ ] Verify plain-text version generation +- [ ] Test unsubscribe/preferences links +- [ ] Send test newsletter to small group +- [ ] Monitor SendGrid dashboard for delivery rates +- [ ] Test across email clients (Gmail, Outlook, Apple Mail) + +--- + +## Security Considerations + +1. **Rate Limiting**: Add rate limiting to send endpoint (max 1 send per 5 minutes) +2. **Admin Only**: Ensure send endpoint requires authentication and admin role +3. **CSRF Protection**: Endpoint uses CSRF token +4. **Input Validation**: Validate tier, subject, and variables +5. **Error Handling**: Don't expose SendGrid errors to client +6. **Logging**: Log all send attempts with admin user ID + +--- + +## Future Enhancements + +1. **Scheduled Sending**: Use cron job or scheduler (e.g., `node-cron`) +2. **A/B Testing**: Send different versions to test groups +3. **Analytics Dashboard**: Track open rates, click rates, unsubscribes +4. **Email Templates Editor**: Visual editor for non-technical users +5. **Subscriber Segmentation**: Advanced targeting beyond just interests +6. **Bounce Handling**: Process SendGrid webhooks for bounces/spam reports + +--- + +## Estimated Implementation Time + +- **Email Service**: 2 hours +- **Controller & Routes**: 1 hour +- **Admin UI JavaScript**: 1 hour +- **Testing & Debugging**: 2 hours +- **Documentation**: 30 minutes + +**Total**: ~6.5 hours + +--- + +## Cost Estimate (SendGrid) + +- Free tier: 100 emails/day +- Essentials: $19.95/month for 50,000 emails/month +- Pro: $89.95/month for 100,000 emails/month + +**Recommendation**: Start with free tier, upgrade as subscriber base grows. + diff --git a/email-templates/README.md b/email-templates/README.md index 0c1e4892..c37d42cf 100644 --- a/email-templates/README.md +++ b/email-templates/README.md @@ -31,3 +31,104 @@ Templates are loaded by the newsletter admin UI and processed with a template en - Brand-consistent (blue gradient header, clean typography) - Clear CTA buttons with good hit targets - Unsubscribe link always visible in footer + +## Template Usage + +### Base Template (`base-template.html`) + +The base template provides the outer HTML structure with header, footer, and styling. It uses these placeholders: + +- `{{email_title}}` - Email subject (for tag) +- `{{header_title}}` - Main header text (e.g., "Tractatus Research Updates") +- `{{header_subtitle}}` - Subtitle text (e.g., "Monthly insights from AI governance research") +- `{{name}}` - Subscriber name +- `{{content_body}}` - HTML content injected here +- `{{tier_name}}` - Newsletter tier name (for footer) +- `{{unsubscribe_link}}` - Unsubscribe URL +- `{{preferences_link}}` - Preferences management URL + +### Content Modules + +Content modules are HTML snippets that get injected into `{{content_body}}`. Example: +- `research-updates-content.html` - Research Updates tier content structure + +### Implementation with Template Engine + +```javascript +const Handlebars = require('handlebars'); +const fs = require('fs'); + +// Load templates +const baseTemplate = fs.readFileSync('email-templates/base-template.html', 'utf8'); +const contentModule = fs.readFileSync('email-templates/research-updates-content.html', 'utf8'); + +// Compile +const compiledContent = Handlebars.compile(contentModule); +const compiledBase = Handlebars.compile(baseTemplate); + +// Render content +const renderedContent = compiledContent({ + highlight_1_title: "...", + highlight_1_summary: "...", + // ... all other variables +}); + +// Render final email +const finalEmail = compiledBase({ + email_title: "Tractatus Research Updates - November 2025", + header_title: "Tractatus Research Updates", + header_subtitle: "Monthly insights from AI governance research", + name: subscriber.name || "there", + content_body: renderedContent, + tier_name: "Research Updates", + unsubscribe_link: `https://agenticgovernance.digital/api/newsletter/unsubscribe?token=${subscriber.token}`, + preferences_link: `https://agenticgovernance.digital/newsletter/preferences?token=${subscriber.token}` +}); +``` + +### Plain Text Version + +Always generate a plain-text version alongside HTML for accessibility and spam filtering: + +```javascript +const htmlToText = require('html-to-text'); +const plainText = htmlToText.convert(finalEmail, { + wordwrap: 80, + ignoreImages: true +}); +``` + +### Testing + +Test emails with [Litmus](https://www.litmus.com/) or [Email on Acid](https://www.emailonacid.com/) across: +- Gmail (desktop, mobile) +- Outlook (2016+, 365) +- Apple Mail (macOS, iOS) +- Yahoo Mail +- Thunderbird + +### Sending + +Use a transactional email service: +- **SendGrid**: https://sendgrid.com/ +- **Mailgun**: https://www.mailgun.com/ +- **AWS SES**: https://aws.amazon.com/ses/ + +Example with SendGrid: + +```javascript +const sgMail = require('@sendgrid/mail'); +sgMail.setApiKey(process.env.SENDGRID_API_KEY); + +await sgMail.send({ + to: subscriber.email, + from: 'research@agenticgovernance.digital', + subject: 'Tractatus Research Updates - November 2025', + html: finalEmail, + text: plainText, + trackingSettings: { + clickTracking: { enable: true }, + openTracking: { enable: true } + } +}); +``` diff --git a/email-templates/base-template.html b/email-templates/base-template.html new file mode 100644 index 00000000..be7c553e --- /dev/null +++ b/email-templates/base-template.html @@ -0,0 +1,65 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <title>{{email_title}} + + + + + + + +
+ + + + + + + + + + + + + + + + + + +
+

{{header_title}}

+

{{header_subtitle}}

+
+

Hi {{name}},

+ + {{content_body}} + +
+

Tractatus AI Safety Framework

+

Architectural constraints for AI safety that preserve human agency

+

+ Website | + Documentation | + GitHub +

+

+ You're receiving this because you subscribed to {{tier_name}}.
+ Unsubscribe | + Update Preferences +

+
+
+ + diff --git a/email-templates/research-updates-content.html b/email-templates/research-updates-content.html new file mode 100644 index 00000000..2fa87cd6 --- /dev/null +++ b/email-templates/research-updates-content.html @@ -0,0 +1,64 @@ + + + +

+ Welcome to this month's research update from the Tractatus AI Safety Framework. Here's what we've been working on and learning. +

+ + +

Research Highlights

+ + + + + + +
+

{{highlight_1_title}}

+

{{highlight_1_summary}}

+ Read More +
+ + + + + + +
+

{{highlight_2_title}}

+

{{highlight_2_summary}}

+ Read More +
+ + +

Key Findings

+ + + + +

Open Questions

+ +

We're still investigating:

+ + + +

+ Have insights on these? Share your thoughts +

+ + + + + + +
+

Want to dive deeper?

+ Explore Our Research +
diff --git a/public/js/components/footer.js b/public/js/components/footer.js index 60f38acf..7d3a067d 100644 --- a/public/js/components/footer.js +++ b/public/js/components/footer.js @@ -79,6 +79,14 @@
  • Media Inquiries
  • Submit Case Study
    • + +

    Subscribe

    + @@ -88,7 +96,6 @@
  • Privacy Policy
  • GDPR Compliance
    • -
  • GitHub
    • diff --git a/scripts/check-csp-violations.js b/scripts/check-csp-violations.js index 1211e70f..e7febb30 100755 --- a/scripts/check-csp-violations.js +++ b/scripts/check-csp-violations.js @@ -42,6 +42,9 @@ function scanForViolations(files = null) { } } + // Exclude email templates - they require inline styles for email client compatibility + htmlFiles = htmlFiles.filter(f => !f.startsWith('email-templates/')); + if (htmlFiles.length === 0) { return []; }