feat: artist cloud drives & external video mirrors (v2.3.0)

2026-09-04 12:08:01 +03:00
parent f603c37c91
commit 639bfc50af
16 changed files with 1369 additions and 10 deletions
@@ -7,6 +7,7 @@ export 'models/artist_work_query.dart';
 export 'models/artist_tag.dart';
 export 'models/artist_link.dart';
 export 'models/artist_announcement.dart';
+export 'models/cloud_media_link.dart';
 export 'models/download_task.dart';
 export 'models/downloaded_media.dart';
 export 'models/favorite.dart';
@@ -0,0 +1,155 @@
+import 'dart:convert';
+import 'package:flutter/material.dart';
+
+enum CloudServiceType {
+  googleDrive,
+  mega,
+  dropbox,
+  pixeldrain,
+  catbox,
+  mediafire,
+  bunkr,
+  gofile,
+  terabox,
+  genericVideo,
+  other,
+}
+
+class CloudMediaLink {
+  const CloudMediaLink({
+    required this.url,
+    required this.service,
+    required this.title,
+    this.directStreamUrl,
+    this.isFolder = false,
+    this.isStreamable = false,
+    this.detectedPassword,
+  });
+
+  final String url;
+  final CloudServiceType service;
+  final String title;
+  final String? directStreamUrl;
+  final bool isFolder;
+  final bool isStreamable;
+  final String? detectedPassword;
+
+  String get serviceName {
+    switch (service) {
+      case CloudServiceType.googleDrive:
+        return 'Google Drive';
+      case CloudServiceType.mega:
+        return 'MEGA';
+      case CloudServiceType.dropbox:
+        return 'Dropbox';
+      case CloudServiceType.pixeldrain:
+        return 'Pixeldrain';
+      case CloudServiceType.catbox:
+        return 'Catbox';
+      case CloudServiceType.mediafire:
+        return 'MediaFire';
+      case CloudServiceType.bunkr:
+        return 'Bunkr';
+      case CloudServiceType.gofile:
+        return 'GoFile';
+      case CloudServiceType.terabox:
+        return 'TeraBox';
+      case CloudServiceType.genericVideo:
+        return 'Direct Video';
+      case CloudServiceType.other:
+        return 'External Mirror';
+    }
+  }
+
+  Color get brandColor {
+    switch (service) {
+      case CloudServiceType.googleDrive:
+        return const Color(0xFF4285F4); // Google Blue
+      case CloudServiceType.mega:
+        return const Color(0xFFD9272E); // Mega Red
+      case CloudServiceType.dropbox:
+        return const Color(0xFF0061FF); // Dropbox Blue
+      case CloudServiceType.pixeldrain:
+        return const Color(0xFF00B4D8); // Cyan
+      case CloudServiceType.catbox:
+        return const Color(0xFFFF6F00); // Amber/Orange
+      case CloudServiceType.mediafire:
+        return const Color(0xFF1E88E5); // MediaFire Blue
+      case CloudServiceType.bunkr:
+        return const Color(0xFF9C27B0); // Purple
+      case CloudServiceType.gofile:
+        return const Color(0xFF26A69A); // Teal
+      case CloudServiceType.terabox:
+        return const Color(0xFF29B6F6); // Light Blue
+      case CloudServiceType.genericVideo:
+        return const Color(0xFFE91E63); // Pink
+      case CloudServiceType.other:
+        return const Color(0xFF78909C); // Slate
+    }
+  }
+
+  IconData get iconData {
+    if (isFolder) return Icons.folder_zip_rounded;
+    if (isStreamable || service == CloudServiceType.genericVideo) {
+      return Icons.play_circle_filled_rounded;
+    }
+    switch (service) {
+      case CloudServiceType.googleDrive:
+        return Icons.add_to_drive_rounded;
+      case CloudServiceType.mega:
+        return Icons.cloud_circle_rounded;
+      case CloudServiceType.dropbox:
+        return Icons.cloud_done_rounded;
+      case CloudServiceType.pixeldrain:
+      case CloudServiceType.catbox:
+      case CloudServiceType.mediafire:
+      case CloudServiceType.bunkr:
+      case CloudServiceType.gofile:
+      case CloudServiceType.terabox:
+        return Icons.cloud_download_rounded;
+      case CloudServiceType.genericVideo:
+        return Icons.videocam_rounded;
+      case CloudServiceType.other:
+        return Icons.link_rounded;
+    }
+  }
+
+  Map<String, dynamic> toJson() => {
+        'url': url,
+        'service': service.name,
+        'title': title,
+        if (directStreamUrl != null) 'directStreamUrl': directStreamUrl,
+        'isFolder': isFolder,
+        'isStreamable': isStreamable,
+        if (detectedPassword != null) 'detectedPassword': detectedPassword,
+      };
+
+  factory CloudMediaLink.fromJson(Map<String, dynamic> json) {
+    final serviceStr = (json['service'] ?? '').toString();
+    final service = CloudServiceType.values.firstWhere(
+      (e) => e.name == serviceStr,
+      orElse: () => CloudServiceType.other,
+    );
+    return CloudMediaLink(
+      url: (json['url'] ?? '').toString(),
+      service: service,
+      title: (json['title'] ?? '').toString(),
+      directStreamUrl: json['directStreamUrl']?.toString(),
+      isFolder: json['isFolder'] == true,
+      isStreamable: json['isStreamable'] == true,
+      detectedPassword: json['detectedPassword']?.toString(),
+    );
+  }
+
+  String encode() => jsonEncode(toJson());
+
+  static CloudMediaLink? tryDecode(String str) {
+    try {
+      final decoded = jsonDecode(str);
+      if (decoded is Map<String, dynamic>) {
+        return CloudMediaLink.fromJson(decoded);
+      }
+    } catch (_) {}
+    return null;
+  }
+}
@@ -1,3 +1,6 @@
+import 'cloud_media_link.dart';
+import '../services/cloud_link_extractor.dart';
+
 class Post {
  const Post({
    required this.id,
@@ -35,6 +38,28 @@ class Post {

  String get cacheKey => '$providerId:$id';

+  List<CloudMediaLink> get cloudLinks {
+    final list = tagGroups['cloud_links'];
+    if (list != null && list.isNotEmpty) {
+      return list
+          .map((e) => CloudMediaLink.tryDecode(e))
+          .whereType<CloudMediaLink>()
+          .toList(growable: false);
+    }
+    if (source != null && source!.isNotEmpty) {
+      return CloudLinkExtractor.extractLinks(source: source);
+    }
+    return const [];
+  }
+
+  String? get commentary {
+    final list = tagGroups['description'];
+    if (list != null && list.isNotEmpty) {
+      return list.first;
+    }
+    return null;
+  }
+
  Post copyWith({
    String? id,
    String? providerId,
@@ -10,6 +10,7 @@ import '../models/post.dart';
 import '../models/post_comment.dart';
 import '../models/provider_health.dart';
 import '../models/top_period_filter.dart';
+import '../services/cloud_link_extractor.dart';
 import 'content_provider.dart';

 class KemonoProvider
@@ -361,6 +362,18 @@ class KemonoProvider
        _dateFromAny(json['added']) ??
        _dateFromAny(json['edited']) ??
        DateTime.fromMillisecondsSinceEpoch(0);
+    final postSource =
+        '$baseUrl/${query.service}/user/${query.artistId}/post/$postId';
+    final rawContent = (json['content'] ?? '').toString();
+    final embed = json['embed'] is Map ? Map<String, dynamic>.from(json['embed']) : null;
+    final cloudLinks = CloudLinkExtractor.extractLinks(
+      content: rawContent,
+      source: postSource,
+      embed: embed,
+    );
+    final cloudLinksJson = cloudLinks.map((e) => e.encode()).toList(growable: false);
+    final cleanDesc = CloudLinkExtractor.cleanCommentary(rawContent);
+
    final files = <Map<String, dynamic>>[];
    final file = json['file'];
    if (file is Map && file.isNotEmpty) {
@@ -370,6 +383,47 @@ class KemonoProvider
    if (attachments is List) {
      files.addAll(attachments.whereType<Map>().map(Map<String, dynamic>.from));
    }
+
+    // Support link-only posts where author uploaded files to cloud drives (MEGA, GDrive, etc.)
+    if (files.isEmpty && (cloudLinks.isNotEmpty || cleanDesc.isNotEmpty)) {
+      final stableId = '${query.service}:${query.artistId}:$postId:0';
+      final firstStreamable =
+          cloudLinks.where((l) => l.isStreamable).firstOrNull;
+      final firstLink = cloudLinks.isNotEmpty ? cloudLinks.first : null;
+      final fileUrl = firstStreamable?.directStreamUrl ?? firstLink?.url ?? '';
+      final tags = [
+        query.service,
+        query.artistName,
+        if (title.trim().isNotEmpty) title.trim(),
+        'cloud_mirror',
+      ];
+      return [
+        Post(
+          id: stableId,
+          providerId: id,
+          providerName: name,
+          previewUrl: '',
+          sampleUrl: '',
+          fileUrl: fileUrl,
+          tags: tags,
+          rating: 'unknown',
+          width: 0,
+          height: 0,
+          source: postSource,
+          createdAt: published,
+          fileType: firstStreamable != null ? 'video' : 'link',
+          score: 0,
+          tagGroups: {
+            'artist': [query.artistName],
+            'meta': [query.service],
+            if (title.trim().isNotEmpty) 'copyright': [title.trim()],
+            if (cloudLinksJson.isNotEmpty) 'cloud_links': cloudLinksJson,
+            if (cleanDesc.isNotEmpty) 'description': [cleanDesc],
+          },
+        ),
+      ];
+    }
+
    var index = 0;
    return files
        .map((file) {
@@ -396,8 +450,7 @@ class KemonoProvider
            rating: 'unknown',
            width: 0,
            height: 0,
-            source:
-                '$baseUrl/${query.service}/user/${query.artistId}/post/$postId',
+            source: postSource,
            createdAt: published,
            fileType: type,
            score: 0,
@@ -405,6 +458,8 @@ class KemonoProvider
              'artist': [query.artistName],
              'meta': [query.service],
              if (title.trim().isNotEmpty) 'copyright': [title.trim()],
+              if (cloudLinksJson.isNotEmpty) 'cloud_links': cloudLinksJson,
+              if (cleanDesc.isNotEmpty) 'description': [cleanDesc],
            },
          );
        })
@@ -12,6 +12,7 @@ import '../models/post.dart';
 import '../models/post_comment.dart';
 import '../models/provider_health.dart';
 import '../models/top_period_filter.dart';
+import '../services/cloud_link_extractor.dart';
 import 'content_provider.dart';

 class PawchiveProvider
@@ -394,6 +395,18 @@ class PawchiveProvider
        _dateFromAny(json['added']) ??
        _dateFromAny(json['edited']) ??
        DateTime.fromMillisecondsSinceEpoch(0);
+    final postSource =
+        '$baseUrl/${query.service}/user/${query.artistId}/post/$postId';
+    final rawContent = (json['content'] ?? '').toString();
+    final embed = json['embed'] is Map ? Map<String, dynamic>.from(json['embed']) : null;
+    final cloudLinks = CloudLinkExtractor.extractLinks(
+      content: rawContent,
+      source: postSource,
+      embed: embed,
+    );
+    final cloudLinksJson = cloudLinks.map((e) => e.encode()).toList(growable: false);
+    final cleanDesc = CloudLinkExtractor.cleanCommentary(rawContent);
+
    final files = <Map<String, dynamic>>[];
    final file = json['file'];
    if (file is Map && file.isNotEmpty) {
@@ -404,6 +417,46 @@ class PawchiveProvider
      files.addAll(attachments.whereType<Map>().map(Map<String, dynamic>.from));
    }

+    // Support link-only posts where author uploaded files to cloud drives (MEGA, GDrive, etc.)
+    if (files.isEmpty && (cloudLinks.isNotEmpty || cleanDesc.isNotEmpty)) {
+      final stableId = '${query.service}:${query.artistId}:$postId:0';
+      final firstStreamable =
+          cloudLinks.where((l) => l.isStreamable).firstOrNull;
+      final firstLink = cloudLinks.isNotEmpty ? cloudLinks.first : null;
+      final fileUrl = firstStreamable?.directStreamUrl ?? firstLink?.url ?? '';
+      final tags = [
+        query.service,
+        query.artistName,
+        if (title.trim().isNotEmpty) title.trim(),
+        'cloud_mirror',
+      ];
+      return [
+        Post(
+          id: stableId,
+          providerId: id,
+          providerName: name,
+          previewUrl: '',
+          sampleUrl: '',
+          fileUrl: fileUrl,
+          tags: tags,
+          rating: 'unknown',
+          width: 0,
+          height: 0,
+          source: postSource,
+          createdAt: published,
+          fileType: firstStreamable != null ? 'video' : 'link',
+          score: 0,
+          tagGroups: {
+            'artist': [query.artistName],
+            'meta': [query.service],
+            if (title.trim().isNotEmpty) 'copyright': [title.trim()],
+            if (cloudLinksJson.isNotEmpty) 'cloud_links': cloudLinksJson,
+            if (cleanDesc.isNotEmpty) 'description': [cleanDesc],
+          },
+        ),
+      ];
+    }
+
    var index = 0;
    return files
        .map((fileMap) {
@@ -436,8 +489,7 @@ class PawchiveProvider
            rating: 'unknown',
            width: 0,
            height: 0,
-            source:
-                '$baseUrl/${query.service}/user/${query.artistId}/post/$postId',
+            source: postSource,
            createdAt: published,
            fileType: type,
            score: 0,
@@ -445,6 +497,8 @@ class PawchiveProvider
              'artist': [query.artistName],
              'meta': [query.service],
              if (title.trim().isNotEmpty) 'copyright': [title.trim()],
+              if (cloudLinksJson.isNotEmpty) 'cloud_links': cloudLinksJson,
+              if (cleanDesc.isNotEmpty) 'description': [cleanDesc],
            },
          );
        })
@@ -0,0 +1,310 @@
+import '../models/cloud_media_link.dart';
+
+class CloudLinkExtractor {
+  static final _urlRegex = RegExp(
+    r'''https?://[^\s<>"{}|\\^`\[\]]+''',
+    caseSensitive: false,
+  );
+
+  static final _htmlLinkRegex = RegExp(
+    r"""<a\s+[^>]*href=["']([^"']+)["'][^>]*>(.*?)</a>""",
+    caseSensitive: false,
+    dotAll: true,
+  );
+
+  static final _passwordRegex = RegExp(
+    r'(?:pass(?:word)?|pw|пароль|пас)(?:[\s:=_-]+)([^\s,;<>"\n\r]{2,50})',
+    caseSensitive: false,
+  );
+
+  /// Extracts any archive password mentioned in the raw text/HTML.
+  static String? extractPassword(String? text) {
+    if (text == null || text.trim().isEmpty) return null;
+    final match = _passwordRegex.firstMatch(text);
+    if (match != null && match.groupCount >= 1) {
+      final pass = match.group(1)?.trim();
+      if (pass != null &&
+          pass.isNotEmpty &&
+          !pass.startsWith('http') &&
+          pass.toLowerCase() != 'none') {
+        return pass;
+      }
+    }
+    return null;
+  }
+
+  /// Extracts all cloud media and video mirror links from text, HTML, and embeds.
+  static List<CloudMediaLink> extractLinks({
+    String? content,
+    String? source,
+    Map<String, dynamic>? embed,
+  }) {
+    final results = <CloudMediaLink>[];
+    final seenUrls = <String>{};
+    final detectedPass = extractPassword(content);
+
+    void addLink(String rawUrl, {String? customTitle}) {
+      final cleanUrl = _cleanUrl(rawUrl);
+      if (cleanUrl == null || seenUrls.contains(cleanUrl)) return;
+      seenUrls.add(cleanUrl);
+
+      final link = _classifyLink(cleanUrl, customTitle: customTitle, password: detectedPass);
+      if (link != null) {
+        results.add(link);
+      }
+    }
+
+    // 1. Process HTML anchors first to preserve anchor text / custom title
+    if (content != null && content.contains('<a ')) {
+      for (final match in _htmlLinkRegex.allMatches(content)) {
+        final href = match.group(1);
+        final rawTitle = match.group(2);
+        final cleanTitle = _stripHtml(rawTitle);
+        if (href != null && href.isNotEmpty) {
+          addLink(href, customTitle: cleanTitle.isNotEmpty ? cleanTitle : null);
+        }
+      }
+    }
+
+    // 2. Process all URLs in content text
+    if (content != null) {
+      for (final match in _urlRegex.allMatches(content)) {
+        final url = match.group(0);
+        if (url != null) addLink(url);
+      }
+    }
+
+    // 3. Process source
+    if (source != null && source.trim().isNotEmpty) {
+      for (final match in _urlRegex.allMatches(source)) {
+        final url = match.group(0);
+        if (url != null) addLink(url);
+      }
+    }
+
+    // 4. Process embed map
+    if (embed != null) {
+      final embedUrl = embed['url']?.toString();
+      if (embedUrl != null && embedUrl.isNotEmpty) {
+        final title = embed['title']?.toString();
+        addLink(embedUrl, customTitle: title);
+      }
+    }
+
+    return results;
+  }
+
+  static String? _cleanUrl(String raw) {
+    var url = raw.trim();
+    // Trim trailing punctuation like ), ., ,, ;, etc.
+    while (url.isNotEmpty && (url.endsWith(')') || url.endsWith('.') || url.endsWith(',') || url.endsWith(';') || url.endsWith('>'))) {
+      url = url.substring(0, url.length - 1);
+    }
+    final parsed = Uri.tryParse(url);
+    if (parsed == null || !parsed.hasScheme || (!parsed.scheme.startsWith('http'))) {
+      return null;
+    }
+    return url;
+  }
+
+  static String _stripHtml(String? html) {
+    if (html == null) return '';
+    return html.replaceAll(RegExp(r'<[^>]*>'), '').trim();
+  }
+
+  static CloudMediaLink? _classifyLink(
+    String url, {
+    String? customTitle,
+    String? password,
+  }) {
+    final uri = Uri.tryParse(url);
+    if (uri == null) return null;
+    final host = uri.host.toLowerCase();
+    final path = uri.path.toLowerCase();
+
+    // Google Drive
+    if (host.contains('drive.google.com')) {
+      final isFolder = path.contains('/folders/') || uri.queryParameters.containsKey('folder');
+      String? directStreamUrl;
+      final fileIdMatch = RegExp(r'/file/d/([a-zA-Z0-9_-]+)').firstMatch(uri.path);
+      final idParam = fileIdMatch?.group(1) ?? uri.queryParameters['id'];
+      if (idParam != null && !isFolder) {
+        directStreamUrl = 'https://drive.google.com/uc?export=download&id=$idParam';
+      }
+      return CloudMediaLink(
+        url: url,
+        service: CloudServiceType.googleDrive,
+        title: customTitle ?? (isFolder ? 'Google Drive Folder' : 'Google Drive File'),
+        directStreamUrl: directStreamUrl,
+        isFolder: isFolder,
+        isStreamable: directStreamUrl != null,
+        detectedPassword: password,
+      );
+    }
+
+    // MEGA
+    if (host.contains('mega.nz') || host.contains('mega.co.nz')) {
+      final isFolder = url.contains('/folder/') || url.contains('/#F!') || url.contains('#F!');
+      return CloudMediaLink(
+        url: url,
+        service: CloudServiceType.mega,
+        title: customTitle ?? (isFolder ? 'MEGA Cloud Folder' : 'MEGA Video / Archive'),
+        directStreamUrl: null, // End-to-end encrypted; launches in MEGA app/browser
+        isFolder: isFolder,
+        isStreamable: false,
+        detectedPassword: password,
+      );
+    }
+
+    // Dropbox
+    if (host.contains('dropbox.com')) {
+      final isFolder = path.contains('/sh/') || path.contains('/folder');
+      var direct = url;
+      if (direct.contains('?dl=0')) {
+        direct = direct.replaceAll('?dl=0', '?raw=1');
+      } else if (!direct.contains('?raw=1') && !isFolder) {
+        direct = direct.contains('?') ? '$direct&raw=1' : '$direct?raw=1';
+      }
+      return CloudMediaLink(
+        url: url,
+        service: CloudServiceType.dropbox,
+        title: customTitle ?? (isFolder ? 'Dropbox Folder' : 'Dropbox Media'),
+        directStreamUrl: isFolder ? null : direct,
+        isFolder: isFolder,
+        isStreamable: !isFolder,
+        detectedPassword: password,
+      );
+    }
+
+    // Pixeldrain
+    if (host.contains('pixeldrain.com')) {
+      final fileMatch = RegExp(r'/u/([a-zA-Z0-9_-]+)').firstMatch(uri.path);
+      final isList = path.contains('/l/');
+      final fileId = fileMatch?.group(1);
+      final direct = fileId != null ? 'https://pixeldrain.com/api/file/$fileId' : null;
+      return CloudMediaLink(
+        url: url,
+        service: CloudServiceType.pixeldrain,
+        title: customTitle ?? (isList ? 'Pixeldrain Gallery/List' : 'Pixeldrain Video'),
+        directStreamUrl: direct,
+        isFolder: isList,
+        isStreamable: direct != null,
+        detectedPassword: password,
+      );
+    }
+
+    // Catbox / Litterbox
+    if (host.contains('catbox.moe')) {
+      final isVideo = path.endsWith('.mp4') || path.endsWith('.webm') || path.endsWith('.mov') || path.endsWith('.m4v');
+      return CloudMediaLink(
+        url: url,
+        service: CloudServiceType.catbox,
+        title: customTitle ?? 'Catbox Upload',
+        directStreamUrl: url,
+        isFolder: false,
+        isStreamable: isVideo,
+        detectedPassword: password,
+      );
+    }
+
+    // MediaFire
+    if (host.contains('mediafire.com')) {
+      final isFolder = path.contains('/folder/') || path.contains('/?');
+      return CloudMediaLink(
+        url: url,
+        service: CloudServiceType.mediafire,
+        title: customTitle ?? (isFolder ? 'MediaFire Folder' : 'MediaFire Download'),
+        directStreamUrl: null,
+        isFolder: isFolder,
+        isStreamable: false,
+        detectedPassword: password,
+      );
+    }
+
+    // Bunkr
+    if (host.contains('bunkr.')) {
+      final isAlbum = path.contains('/a/') || path.contains('/v/');
+      return CloudMediaLink(
+        url: url,
+        service: CloudServiceType.bunkr,
+        title: customTitle ?? (isAlbum ? 'Bunkr Album' : 'Bunkr Media'),
+        directStreamUrl: null,
+        isFolder: isAlbum,
+        isStreamable: false,
+        detectedPassword: password,
+      );
+    }
+
+    // GoFile
+    if (host.contains('gofile.io')) {
+      return CloudMediaLink(
+        url: url,
+        service: CloudServiceType.gofile,
+        title: customTitle ?? 'GoFile Folder / Archive',
+        directStreamUrl: null,
+        isFolder: true,
+        isStreamable: false,
+        detectedPassword: password,
+      );
+    }
+
+    // TeraBox
+    if (host.contains('terabox') || host.contains('1024tera') || host.contains('dubox')) {
+      return CloudMediaLink(
+        url: url,
+        service: CloudServiceType.terabox,
+        title: customTitle ?? 'TeraBox Cloud Share',
+        directStreamUrl: null,
+        isFolder: true,
+        isStreamable: false,
+        detectedPassword: password,
+      );
+    }
+
+    // Direct Video Link (MP4, WEBM, MKV, MOV)
+    if (path.endsWith('.mp4') || path.endsWith('.webm') || path.endsWith('.mov') || path.endsWith('.m4v') || path.endsWith('.mkv')) {
+      final filename = uri.pathSegments.isNotEmpty ? uri.pathSegments.last : 'Video Stream';
+      return CloudMediaLink(
+        url: url,
+        service: CloudServiceType.genericVideo,
+        title: customTitle ?? filename,
+        directStreamUrl: url,
+        isFolder: false,
+        isStreamable: true,
+        detectedPassword: password,
+      );
+    }
+
+    // Ignore known booru and search engine domains
+    if (host.contains('kemono') ||
+        host.contains('coomer') ||
+        host.contains('danbooru') ||
+        host.contains('gelbooru') ||
+        host.contains('google.com') && !host.contains('drive') ||
+        host.contains('twitter.com') ||
+        host.contains('x.com') ||
+        host.contains('patreon.com') ||
+        host.contains('fanbox.cc') ||
+        host.contains('pixiv.net')) {
+      return null;
+    }
+
+    return null;
+  }
+
+  /// Extracts clean text (stripping HTML tags) suitable for author commentary display.
+  static String cleanCommentary(String? rawHtml) {
+    if (rawHtml == null || rawHtml.trim().isEmpty) return '';
+    var text = rawHtml
+        .replaceAll(RegExp(r'<br\s*\/?>', caseSensitive: false), '\n')
+        .replaceAll(RegExp(r'<\/p>', caseSensitive: false), '\n\n')
+        .replaceAll(RegExp(r'<[^>]*>'), '')
+        .replaceAll('&nbsp;', ' ')
+        .replaceAll('&amp;', '&')
+        .replaceAll('&lt;', '<')
+        .replaceAll('&gt;', '>')
+        .replaceAll('&quot;', '"')
+        .replaceAll('&#39;', "'");
+    return text.trim();
+  }
+}