works_ feb 21, 2025, 30263 bytes


::::: port init


(custom-set-variables
 ;; custom-set-variables was added by Custom.
 ;; If you edit it by hand, you could mess it up, so be careful.
 ;; Your init file should contain only one such instance.
 ;; If there is more than one, they won't work right.
 '(backup-by-copying t)
 '(custom-enabled-themes '(modus-operandi-tinted))
 '(delete-old-versions t)
 '(global-display-line-numbers-mode t)
 '(indent-tabs-mode nil)
 '(initial-buffer-choice "/sdcard/")
 '(kept-new-versions 3)
 '(kept-old-versions 3)
 '(make-backup-files t)
 '(modifier-bar-mode t)
 '(pop-up-frames 'graphic-only)
 '(tab-width 4)
 '(tool-bar-position 'bottom)
 '(touch-screen-display-keyboard t)
 '(touch-screen-extend-selection t)
 '(touch-screen-word-select t)
 '(truncate-lines nil)
 '(version-control t))

(custom-set-faces
 ;; custom-set-faces was added by Custom.
 ;; If you edit it by hand, you could mess it up, so be careful.
 ;; Your init file should contain only one such instance.
 ;; If there is more than one, they won't work right.
 )
 
 ;; removed settings
 ;;  '(desktop-save-mode t)


;; https://stackoverflow.com/a/18330742 

(defvar --backup-directory (concat user-emacs-directory "backups"))
(if (not (file-exists-p --backup-directory))
        (make-directory --backup-directory t))
(setq backup-directory-alist `(("." . ,--backup-directory)))


;; batch replacements

(defun batch-replace-from-file ()
  "Perform batch replacements using data from an input file and modify the target file in place."
  (interactive)
  ;; Prompt for input and target file paths
  (let ((input-file (read-file-name "Enter the input file path (search-replace pairs): "))
        (target-file (read-file-name "Enter the target file path (file to modify): "))
        (search-replace-pairs '()))
    
    ;; Read the input file and parse search-replace pairs
    (with-temp-buffer
      (insert-file-contents input-file)
      (goto-char (point-min))
      (while (re-search-forward "^\\(.*?\\)\\s-*,\\(.*\\)$" nil t)
        (let ((search-term (match-string 1))
              (replace-term (match-string 2)))
          (push (cons search-term replace-term) search-replace-pairs))))
    
    ;; Reverse the list to maintain original order
    (setq search-replace-pairs (reverse search-replace-pairs))
    
    ;; Read the target file and perform replacements
    (with-temp-buffer
      (insert-file-contents target-file)
      (goto-char (point-min))
      
      ;; Perform replacements for each search-replace pair
      (dolist (pair search-replace-pairs)
        (let ((search-term (car pair))
              (replace-term (cdr pair)))
          (goto-char (point-min))
          (while (re-search-forward (concat "\\b" (regexp-quote search-term) "\\b") nil t)
            (replace-match replace-term))))
      
      ;; Write the modified content back to the target file
      (write-region (point-min) (point-max) target-file))
    
    (message "Batch replacements completed. Changes saved to %s" target-file)))


::::: termux init

(custom-set-variables
 ;; custom-set-variables was added by Custom.
 ;; If you edit it by hand, you could mess it up, so be careful.
 ;; Your init file should contain only one such instance.
 ;; If there is more than one, they won't work right.
 '(backup-by-copying t)
 '(default-frame-alist '((fullwidth) (height . 32)))
 '(delete-old-versions t)
 '(desktop-save-mode t)
 '(global-display-line-numbers-mode t)
 '(indent-tabs-mode nil)
 '(initial-buffer-choice "/sdcard/")
 '(kept-new-versions 3)
 '(kept-old-versions 3)
 '(make-backup-files t)
 '(pop-up-frames 'graphic-only)
 '(tab-width 4)
 '(truncate-lines t)
 '(version-control t))

(custom-set-faces
 ;; custom-set-faces was added by Custom.
 ;; If you edit it by hand, you could mess it up, so be careful.
 ;; Your init file should contain only one such instance.
 ;; If there is more than one, they won't work right.
 )

;; https://stackoverflow.com/a/18330742 
(defvar --backup-directory (concat user-emacs-directory "backups"))
(if (not (file-exists-p --backup-directory))
        (make-directory --backup-directory t))
(setq backup-directory-alist `(("." . ,--backup-directory)))


::::: perl scripts:
::::: for extracting citation text
::::: with or without dois found in references
::::: above each #!/usr/bin/perl there will be numbers
::::: numbers are issn numbers

> 2321-1326 | 2394-112X ! 10.17727/JMSR.2022/10-36

#!/usr/bin/perl
# 2321-1326 | 2394-112X ! 10.17727/JMSR.2022/10-36

use strict;
use warnings;
use utf8;
use open ':std', ':encoding(UTF-8)';
use HTML::TreeBuilder; # For parsing HTML
use HTML::Strip;       # For stripping HTML tags

# Hardcoded file paths
my $input_file  = "/storage/emulated/0/ccbys/up/oldoi.txt";  # Input file path
my $output_file = "/storage/emulated/0/ccbys/up/otdoi.txt"; # Output file path
my $debug_file  = "/storage/emulated/0/ccbys/up/dgdoi.txt"; # Debug file path

# Open input, output, and debug files
open my $in_fh,  '<', $input_file  or die "Cannot open input file: $!";
open my $out_fh, '>', $output_file or die "Cannot open output file: $!";
open my $dbg_fh, '>', $debug_file  or die "Cannot open debug file: $!";

my $citation_count = 0;

# Read the entire input file into a scalar
my $html_content = do { local $/; <$in_fh> };

# Parse the HTML content using HTML::TreeBuilder
my $tree = HTML::TreeBuilder->new_from_content($html_content);

# Initialize HTML::Strip for removing HTML tags
my $hs = HTML::Strip->new();

# Find all <p> tags in the HTML
for my $p_tag ($tree->find_by_tag_name('p')) {
    # Get the text content of the <p> tag
    my $citation = $p_tag->as_text();

    # Remove HTML tags using HTML::Strip
    $citation = $hs->parse($citation);
    $hs->eof;

    # Debug: Print the processed citation
    print $dbg_fh "Processing citation: $citation\n";

    # Skip empty citations (e.g., <p>&nbsp;</p>)
    next if $citation =~ /^\s*$/;

    # Remove [1], [2], etc., from the citation text
    # Updated regex to handle cases like [11], [12], etc.
    $citation =~ s/^\s*\[\d+\]\s*//;

    # Extract DOI if present
    my $doi = '';
    if ($citation =~ /https:\/\/doi\.org\/([^\s]+)/) {
        $doi = $1;
        print $dbg_fh "Extracted DOI: $doi\n";
    }

    # Format the output
    my $output_line;
    if ($doi) {
        # If DOI is present, include DOI URLs
        $output_line = sprintf("q%d,    {%d} %s https://chooser.crossref.org/?doi=%s ! https://crossmark.crossref.org/dialog?doi=%s\n",
                               $citation_count + 1, $citation_count + 1, $citation, $doi, $doi);
    } else {
        # If no DOI is present, exclude DOI URLs
        $output_line = sprintf("q%d,    {%d} %s\n",
                               $citation_count + 1, $citation_count + 1, $citation);
    }

    # Write to output file
    print $out_fh $output_line;
    print $dbg_fh "Wrote to output file: $output_line";

    # Increment citation count
    $citation_count++;
}

# Clean up
$tree->delete;
close $in_fh;
close $out_fh;
close $dbg_fh;

print "Processing complete. Output written to $output_file. Debug information written to $debug_file.\n";


> 0976-5042 | 0976-5050, 0973-0508 | 2213-3739

#!/usr/bin/perl
# 0976-5042 | 0976-5050 ! 10.1055/s-0042-1759510
# 0973-0508 | 2213-3739 ! 10.1055/s-0041-1739477

use strict;
use warnings;
use utf8;
use open ':std', ':encoding(UTF-8)';
use HTML::Strip;

# Define input, output, and debug file paths
my $input_file  = "/storage/emulated/0/ccbys/up/oldoi.txt" || die "Usage: $0 <input_file> <output_file> <debug_file>\n";
my $output_file = "/storage/emulated/0/ccbys/up/otdoi.txt" || die "Usage: $0 <input_file> <output_file> <debug_file>\n";
my $debug_file  = "/storage/emulated/0/ccbys/up/dgdoi.txt" || die "Usage: $0 <input_file> <output_file> <debug_file>\n";

# Open files
open(my $in_fh,  '<', $input_file)  or die "Could not open input file '$input_file' $!";
open(my $out_fh, '>', $output_file) or die "Could not open output file '$output_file' $!";
open(my $dbg_fh, '>', $debug_file)  or die "Could not open debug file '$debug_file' $!";

# Read the entire file content
my $file_content = do { local $/; <$in_fh> };

# Initialize HTML stripper
my $hs = HTML::Strip->new();

# Extract each citation block
my @citation_blocks = split(/<\/li>/, $file_content);

# Process each citation block
my $count = 1;
foreach my $block (@citation_blocks) {
    # Skip empty blocks
    next unless $block =~ /\S/;

    # Extract the DOI from <a data-url="https://doi.org/..."> within <div class="referenceLinks">
    my ($doi) = $block =~ /<div class="referenceLinks">.*?<a data-url="https:\/\/doi\.org\/(10\.\d+\/[\w\.\-]+)"/s;
    $doi //= '';  # Default to empty string if no DOI found

    # Strip HTML tags and clean the text (only before <div class="referenceLinks">)
    my ($citation_text) = $block =~ /(.*?)<div class="referenceLinks">/s;
    $citation_text //= $block;  # Fallback to the entire block if <div class="referenceLinks"> is not found
    my $clean_text = $hs->parse($citation_text);
    $hs->eof;

    # Remove non-breaking spaces and replace Unicode equivalents
    $clean_text =~ s/\x{00A0}/ /g;  # Non-breaking space
    $clean_text =~ s/&amp;/&/g;     # Replace &amp; with &
    $clean_text =~ s/&lt;/</g;      # Replace &lt; with <
    $clean_text =~ s/&gt;/>/g;      # Replace &gt; with >
    $clean_text =~ s/&quot;/"/g;    # Replace &quot; with "
    $clean_text =~ s/&apos;/'/g;    # Replace &apos; with '

    # Remove the digit on a separate line and combine multi-line names into a single line
    $clean_text =~ s/^\s*\d+\s*$//mg;  # Remove lines with only a digit
    $clean_text =~ s/\n\s*//g;         # Combine multi-line names into a single line

    # Format the output
    my $formatted_output;
    if ($doi) {
        # If DOI is found, append the DOI URLs
        $formatted_output = sprintf(
            "q%d,    {%d} %s https://chooser.crossref.org/?doi=%s ! https://crossmark.crossref.org/dialog?doi=%s\n",
            $count, $count, $clean_text, $doi, $doi
        );
    } else {
        # If no DOI is found, do not append the DOI URLs
        $formatted_output = sprintf(
            "q%d,    {%d} %s\n",
            $count, $count, $clean_text
        );
    }

    # Write to output file
    print $out_fh $formatted_output;

    # Write to debug file
    print $dbg_fh "Original Block:\n$block\n";
    print $dbg_fh "Cleaned Text:\n$clean_text\n";
    print $dbg_fh "DOI: $doi\n" if $doi;

    $count++;
}

# Close files
close($in_fh);
close($out_fh);
close($dbg_fh);

print "Processing complete. Output saved to $output_file. Debug information saved to $debug_file.\n";


>  0973-7510 | 2581-690X, 10.22207/JPAM.14.2.41

#!/usr/bin/perl
# 0973-7510 | 2581-690X, 10.22207/JPAM.14.2.41

use strict;
use warnings;
use utf8;
use open ':std', ':encoding(UTF-8)';
use HTML::Entities;  # Module to decode HTML entities

# Command line arguments for input, output, and debug files
my ($input_file, $output_file, $debug_file) = @ARGV;

# Set default output and debug file names if not provided
$output_file //= "output.txt";
$debug_file //= "debug.txt";

# Open input file
open my $in_fh, '<', "/storage/emulated/0/ccbys/up/oldoi.txt" or die "Cannot open input file: $!";

# Open output file
open my $out_fh, '>', "/storage/emulated/0/ccbys/up/otdoi.txt" or die "Cannot open output file: $!";

# Open debug file (optional)
open my $dbg_fh, '>', "/storage/emulated/0/ccbys/up/dgdoi.txt" or die "Cannot open debug file: $!" if $debug_file;

my $citation_number = 1;
my $citation_text = '';
my $doi_number = '';

while (my $line = <$in_fh>) {
    # Remove non-breaking spaces and Unicode equivalents
    $line =~ s/\x{00A0}/ /g;  # Non-breaking space
    $line =~ s/\x{202F}/ /g;  # Narrow non-breaking space

    # Check if the line starts with <li>
    if ($line =~ /<li>/) {
        # Reset citation text and DOI number for new citation
        $citation_text = '';
        $doi_number = '';
    }

    # Append the line to the citation text, removing line breaks
    $citation_text .= $line unless $line =~ /<a href="https:\/\/doi\.org\//;

    # Extract DOI number if the line contains a DOI URL
    if ($line =~ /<a href="https:\/\/doi\.org\/([^"]+)"/) {
        $doi_number = $1;
    }

    # Check if the line ends with </li>
    if ($line =~ /<\/li>/) {
        # Remove HTML tags and extra spaces from citation text
        $citation_text =~ s/<[^>]+>//g;
        $citation_text =~ s/\s+/ /g;
        $citation_text =~ s/^\s+|\s+$//g;

        # Decode HTML entities (e.g., &#8211; to –)
        decode_entities($citation_text);

        # Format the output line
        my $output_line = sprintf("q%d,    {%d} %s https://chooser.crossref.org/?doi=%s ! https://crossmark.crossref.org/dialog?doi=%s\n",
            $citation_number, $citation_number, $citation_text, $doi_number, $doi_number);

        # Write to output file
        print $out_fh $output_line;

        # Write to debug file if specified
        print $dbg_fh $output_line if $debug_file;

        # Increment citation number
        $citation_number++;
    }
}

# Close file handles
close $in_fh;
close $out_fh;
close $dbg_fh if $debug_file;

print "Processing complete. Output written to $output_file\n";
print "Debug output written to $debug_file\n" if $debug_file;


::::: bash wget script to sequential download
::::: webpages with unique filenames

#!/bin/bash

# Check if an output directory argument is provided
if [ -z "$1" ]; then
    echo "Usage: $0 <output_directory>"
    exit 1
fi

# Define the output directory
output_dir="$1"

# Create the output directory if it doesn't exist
mkdir -p "$output_dir"

index=1

# Function to check if a URL should be excluded based on file extension
should_exclude() {
    local url="$1"
    [[ "$url" =~ \.(jpg|png|gif|pdf|epub)$ ]]  # Match URLs with specific extensions
}

# Read URLs from the file and process them
while IFS= read -r url; do
    # If the URL should not be excluded, start the download
    if ! should_exclude "$url"; then
        wget -O "$output_dir/index_$index.html" "$url"
        ((index++))
    else
        echo "Skipping URL (excluded file type): $url"
    fi
done < urls.txt


::::: find doi prefix [ "prefix": ]

https://api.crossref.org/works?query={journal name}&rows=1


::::: find landing page url of a doi
::::: [ "resource": { "primary": { ]

https://api.production.crossref.org/works/DOI


::::: search openalex with keyword
::::: doi prefix and source
::::: replace 00000 with doi prefix
::::: replace S00000 with source id

https://api.openalex.org/works?page=1&filter=type:types/article,doi_starts_with:000000,primary_location.source.id:S00000,default.search:women&sort=relevance_score:desc&per_page=10


::::: using lynx find unique urls in webpage
::::: replace URL

lynx -dump -listonly URL | grep -oP 'https?://[^\s]+' | sort | uniq


::::: using wget download webpage
::::: excluding images, pdfs
::::: replace URL

wget -r -l1 --no-parent --reject "*.pdf,*.jpg,*.jpeg,*.png,*.gif,*.bmp,*.svg,*.webp" URL


::::: convert html table to text

html2text -width 500 -nobs -utf8 -o table.txt  table.html


::::: retrieve apa citation style
::::: from doi

https://citation.doi.org/format?doi=QQQQQ&style=apa&lang=en-GB


::::: sunset code

::::: bash scripts:
::::: for extracting citation text
::::: with or without dois found in references
::::: above each #!/bin/bash there will be numbers
::::: numbers are issn numbers

> 2321-9971 | 2347-4688

#!/bin/bash

# extract citations with or without dois
# issn: 2321-9971 | 2347-4688

# Input and output file paths
input_file="path to input file"
output_file="path to output file"

# Process the file
awk '
BEGIN {
    # Regular expression to match DOI links
    doi_regex = "https://doi.org/([0-9]+\\.[^\">]+)"
    buffer = "" # Initialize a buffer for concatenating lines
}
{
    # Accumulate lines into the buffer until a closing </li>
    buffer = buffer $0
    if ($0 ~ /<\/li>/) {
        # Process the complete buffer
        line = buffer
        buffer = "" # Reset buffer for the next <li>

        # Extract and remove DOI
        doi = ""
        if (match(line, doi_regex, match_arr)) {
            doi = match_arr[1]  # Extract DOI (strip "https://doi.org/")
            gsub(doi_regex, "", line) # Remove DOI link from the line
        }

        # Remove all remaining HTML tags
        gsub(/<\/?[^>]+>/, "", line)
        # Replace HTML entities with their plain text equivalents
        gsub(/&amp;/, "&")
        # Remove "CrossRef" text
        gsub(/CrossRef/, "", line)
        # Remove non-breaking spaces and right-to-left marks (RLM)
        gsub(/\xA0|\u200E|\u200F/, "")
        # Trim leading and trailing spaces
        gsub(/^[ \t]+|[ \t]+$/, "", line)

        # Replace appended DOI with URLs
        if (doi != "") {
            gsub("doi:" doi, "", line) # Remove "doi: DOI_NUMBER" if present
            gsub(/[ ]+/, " ") # Ensure no extra spaces
            line = line " https://chooser.crossref.org/?doi=" doi " ! https://crossmark.crossref.org/dialog?doi=" doi
        }

        # Output the cleaned line if non-empty
        if (length(line) > 0) {
            print line
        }
    }
}' "$input_file" > "$output_file"

echo "Processing completed. Output saved to $output_file."



> DE 2321-1326 | 2394-112X, 2322-0007 | 2347-467X, 2581-9933 | 2581-9178

#!/usr/bin/perl
# extract citations with or without dois
# issn: 2321-1326 | 2394-112X, 2322-0007 | 2347-467X

use strict;
use warnings;
use utf8;

# Command-line argument for output file
my $output_file = 'pdoi.txt';
unless ($output_file) {
    print "Output file not specified. Please enter the output file name: ";
    $output_file = <STDIN>;
    chomp($output_file);  # Remove newline character
    unless ($output_file) {
        die "Error: Output file must be specified.\n";
    }
}

# Debug file for troubleshooting
my $debug_file = 'debug.txt';
open(my $debug_fh, '>:encoding(UTF-8)', $debug_file) or die "Could not open debug file '$debug_file' $!";

# Read the file content
my $filename = 'doi.txt';
open(my $fh, '<:encoding(UTF-8)', $filename) or die "Could not open file '$filename' $!";

# Read the entire file content
my $content = do { local $/; <$fh> };

# Close the file handle
close($fh);

# Open the output file for writing
open(my $out_fh, '>:encoding(UTF-8)', $output_file) or die "Could not open output file '$output_file' $!";

# Counter for sequential numbering
my $counter = 1;

# Regular expression to match <p> and <li> content and extract citation
while ($content =~ /<(?:p|li)[^>]*>(.*?)<\/(?:p|li)>/gs) {
    my $citation = $1;

    # Debugging: Print the raw citation to debug file
    print $debug_fh "Raw citation: $citation\n";

    # Skip if the citation is empty or contains only &nbsp;
    next if $citation =~ /^\s*&nbsp;\s*$/;

    # Remove [x] where x is a digit
    $citation =~ s/\[\d+\]//g;

    # Initialize DOI URL
    my $doi_url = "";

    # Extract DOI from <a href="https://doi.org/..."> tags within <p> or <li>
    if ($citation =~ /<a [^>]*href="https:\/\/doi\.org\/([^\s"]+)"/) {
        my $doi = $1;

        # Debugging: Print the extracted DOI to debug file
        print $debug_fh "Extracted DOI: $doi\n";

        # Construct DOI URLs
        $doi_url = " https://chooser.crossref.org/?doi=$doi ! https://crossmark.crossref.org/dialog?doi=$doi";

        # Debugging: Print the constructed DOI URLs to debug file
        print $debug_fh "Constructed DOI URLs: $doi_url\n";
    }

    # Remove HTML tags and non-breaking spaces
    $citation =~ s/<[^>]+>//g;  # Remove all HTML tags
    $citation =~ s/&amp;quot;/"/g;  # Replace &amp;quot; with "
    $citation =~ s/&amp;amp;/&/g;  # Replace &amp;amp; with &
    $citation =~ s/\x{200E}|\x{200F}//g;  # Remove left-to-right and right-to-left marks
    $citation =~ s/\s+/ /g;  # Normalize spaces

    # Replace Unicode equivalents with their actual characters
    $citation =~ s/&ndash;/–/g;  # Replace &ndash; with –
    $citation =~ s/&rsquo;/’/g;  # Replace &rsquo; with ’
    $citation =~ s/&ldquo;/“/g;  # Replace &ldquo; with “
    $citation =~ s/&rdquo;/”/g;  # Replace &rdquo; with ”
    $citation =~ s/&hellip;/…/g; # Replace &hellip; with …
    $citation =~ s/&mdash;/—/g;  # Replace &mdash; with —

    # Remove unwanted text: "Crossref", "PubMed", "Search in Google Scholar", and "CrossRef"
    $citation =~ s/\bCrossref\b//g;  # Remove standalone "Crossref"
    $citation =~ s/\bPubMed\b//g;    # Remove standalone "PubMed"
    $citation =~ s/\bSearch in Google Scholar\b//g;  # Remove standalone "Search in Google Scholar"
    $citation =~ s/\bCrossRef\b//g;  # Remove standalone "CrossRef"

    # Debugging: Print the cleaned citation to debug file
    print $debug_fh "Cleaned citation: $citation\n";

    # Format the output as "q#,    {#} citation text"
    my $formatted_output = "q$counter,    {$counter} $citation$doi_url\n";

    # Debugging: Print the final formatted output to debug file
    print $debug_fh "Formatted output: $formatted_output\n";

    # Print the formatted output to the output file
    print $out_fh $formatted_output;

    # Increment the counter
    $counter++;
}

# Close the output file handle
close($out_fh);

# Close the debug file handle
close($debug_fh);

print "Extraction complete. Output saved to '$output_file'.\n";
print "Debug information saved to '$debug_file'.\n";


> CH 2394-0565 | 0973-3469

#!/bin/bash
# extract citations with or without dois
# issn: 2394-0565 | 0973-3469
# everything is on 1 line

# Output file
OUTPUT_FILE="output.txt"

# Ensure each <li> block is on a new line and extract content between <li> and </li>
sed 's/<\/li>/<\/li>\n/g' doi.txt | sed -n 's/.*<li>\(.*\)<\/li>.*/\1/p' | awk '
BEGIN {
    OFS = "";  # No automatic field separator
}
{
    # Extract DOI if available
    doi = "";
    if (match($0, /https:\/\/doi\.org\/([^"]+)/, arr)) {
        doi = " https://chooser.crossref.org/?doi=" arr[1] " ! https://crossmark.crossref.org/dialog?doi=" arr[1];
    }

    # Remove everything after <br> (including DOI links)
    sub(/<br>.*/, "");

    # Print formatted output with comma, 4 spaces, sequential {#}, and space before DOI
    printf "q%d,    {%d} %s%s\n", NR, NR, $0, doi;
}' > "$OUTPUT_FILE"

# Check if output is empty
if [ ! -s "$OUTPUT_FILE" ]; then
    echo "Error: No citations extracted. Check if doi.txt is formatted correctly."
    exit 1
fi

echo "Output saved to $OUTPUT_FILE"


::::: perl scripts:
::::: for extracting citation text
::::: with or without dois found in references
::::: above each #!/usr/bin/perl there will be numbers
::::: numbers are issn numbers

> CH 2394-0565 | 0973-3469

#!/usr/bin/perl
# extract citations with or without dois
# issn: 2394-0565 | 0973-3469
# everything is on 1 line

use strict;
use warnings;
use open qw(:std :utf8);  # Ensure UTF-8 compatibility

# Input and output files
my $input_file  = "doi.txt";
my $output_file = "output.txt";

open(my $in, "<:utf8", $input_file) or die "Cannot open $input_file: $!";
open(my $out, ">:utf8", $output_file) or die "Cannot open $output_file: $!";

# Read entire file into a string
my $content = do { local $/; <$in> };
close($in);

# Ensure each <li> block is on a new line
$content =~ s/<\/li>/<\/li>\n/g;

# Extract citations between <li> and </li>
my @citations = ($content =~ /<li>(.*?)<\/li>/sg);

my $count = 1;
foreach my $citation (@citations) {
    # Extract DOI numbers before modifying the citation
    my %dois;
    while ($citation =~ /(https:\/\/doi\.org\/[^"\s]+)/g) {
        $dois{$1} = 1;  # Store unique DOIs in a hash
    }

    # Remove all DOI links from the citation text
    $citation =~ s/https:\/\/doi\.org\/[^"\s]+//g;

    # Remove everything after <br>
    $citation =~ s/<br>.*//s;

    # Format DOI URLs (ensure uniqueness)
    my $doi_str = "";
    if (%dois) {
        my @formatted_dois = map { "https://chooser.crossref.org/?doi=" . substr($_, 16) . " ! https://crossmark.crossref.org/dialog?doi=" . substr($_, 16) } keys %dois;
        $doi_str = " " . join(" ", @formatted_dois);
    }

    # Prepend q1, (q1 comma 4 spaces) and {#} before the citation text
    print $out sprintf("q%d,    {%d} %s%s\n", $count, $count, $citation, $doi_str);
    
    $count++;
}

close($out);
print "Output saved to $output_file\n";

> DE extract html table 2 markdown table

use strict;
use warnings;
use HTML::TreeBuilder;
use File::Slurp;
use Encode qw(encode decode);

# Input and output file paths
my $input_file = 'tx.html';
my $output_file = 'tx.md';
my $debug_file = 'dtx.txt';

# Read the HTML content
my $html_content = read_file($input_file);

# Ensure UTF-8 encoding
$html_content = decode('UTF-8', $html_content);

# Parse the HTML content
my $tree = HTML::TreeBuilder->new_from_content($html_content);

# Find the table
my $table = $tree->look_down('_tag', 'table');

# Initialize variables to store the table data
my @table_data;

# Function to clean invisible characters
sub clean_text {
    my $text = shift;
    return unless defined $text;

    # Replace non-breaking spaces (&nbsp;) and other invisible characters
    $text =~ s/\xA0/ /g;  # Replace non-breaking spaces with regular spaces
    $text =~ s/\x{200B}//g;  # Remove zero-width spaces
    $text =~ s/\x{FEFF}//g;  # Remove byte order marks (BOM)
    $text =~ s/\s+/ /g;  # Replace multiple spaces with a single space
    $text =~ s/^\s+|\s+$//g;  # Trim leading and trailing spaces

    return $text;
}

# Process each row
foreach my $row ($table->look_down('_tag', 'tr')) {
    my @cells = $row->look_down('_tag', 'td');
    my @row_data;

    # Extract the first column (e.g., "Age group", "Ever pregnant", etc.)
    if ($cells[0]->attr('rowspan')) {
        my $first_column = clean_text($cells[0]->as_trimmed_text);
        push @row_data, $first_column;

        # Process the remaining cells as rows sharing the first column
        for (my $i = 1; $i < scalar(@cells); $i++) {
            push @row_data, clean_text($cells[$i]->as_trimmed_text);
        }

        # Add the row data to the table
        push @table_data, \@row_data;
    }
    else {
        # If no rowspan, treat as a continuation of the previous row
        if (@table_data) {
            my $first_column = $table_data[-1][0]; # Use the first column from the previous row
            push @row_data, $first_column;

            # Process all cells as rows sharing the first column
            foreach my $cell (@cells) {
                push @row_data, clean_text($cell->as_trimmed_text);
            }

            # Add the row data to the table
            push @table_data, \@row_data;
        } else {
            # Handle the case where @table_data is empty (first row without rowspan)
            foreach my $cell (@cells) {
                push @row_data, clean_text($cell->as_trimmed_text);
            }
            push @table_data, \@row_data;
        }
    }
}

# Determine the maximum number of columns
my $max_columns = 0;
foreach my $row (@table_data) {
    $max_columns = scalar(@$row) if scalar(@$row) > $max_columns;
}

# Determine the maximum text length in each column
my @max_lengths;
for (my $i = 0; $i < $max_columns; $i++) {
    my $max_length = 0;
    foreach my $row (@table_data) {
        if (defined $row->[$i] && length($row->[$i]) > $max_length) {
            $max_length = length($row->[$i]);
        }
    }
    push @max_lengths, $max_length;
}

# Write the table data to the output file in Markdown format
open(my $out_fh, '>:encoding(UTF-8)', $output_file) or die "Could not open file '$output_file' $!";

# Write the table rows
foreach my $row (@table_data) {
    my @formatted_row;
    for (my $i = 0; $i < scalar(@$row); $i++) {
        my $text = $row->[$i] // '';
        $text .= ' ' x ($max_lengths[$i] - length($text)); # Pad with spaces
        push @formatted_row, $text;
    }
    print $out_fh '| ' . join(' | ', @formatted_row) . " |\n";
}

# Close the output file
close($out_fh);

# Debugging: Write the parsed data to a debug file
open(my $debug_fh, '>:encoding(UTF-8)', $debug_file) or die "Could not open file '$debug_file' $!";
print $debug_fh "Table Data:\n";
foreach my $row (@table_data) {
    print $debug_fh join(', ', @$row) . "\n";
}
close($debug_fh);

# Clean up the tree
$tree->delete;

> 2581-9933 | 2581-9178, 0976-5042 | 0976-5050

#!/bin/bash

# extract citations with or without dois
# issn: 2581-9933 | 2581-9178, 0976-5042 | 0976-5050

# Input and output file paths
input_file="path to input file"
output_file="path to output file"

# Extract and clean the required data
awk '
BEGIN {
    RS="<li>";  # Treat <li> tags as record separators
    FS="<div class=\"referenceLinks\">";  # Split each record by the end marker
    count = 1;  # Sequential counter
}
{
    entry = "";
    doi_number = "";

    # Match entries starting with <a name="ALPHANUMERIC-NUMBER"></a><strong>...</strong>
    if ($1 ~ /<a name="[A-Za-z0-9]+-[0-9]+"><\/a><strong>[0-9]+<\/strong>/) {
        entry = $1;
        gsub(/<[^>]*>/, "", entry);  # Remove all HTML tags
        gsub(/[ \t\n]+/, " ", entry);  # Replace multiple spaces or newlines with a single space
        gsub(/^[ \t]*[0-9]+[ \t]*/, "", entry);  # Remove leading numbers and spaces
        gsub(/^[ \t]+|[ \t]+$/, "", entry);  # Trim leading and trailing spaces
    }

    # Extract DOI number using a more compatible method
    if ($2 ~ /data-url="https:\/\/doi.org\//) {
        doi_number = $2;
        sub(/.*data-url="https:\/\/doi.org\//, "", doi_number);  # Remove prefix
        sub(/".*/, "", doi_number);  # Remove trailing quote and extra text
    }

    # Print the entry with the correct formatting
    if (entry != "") {
        if (doi_number != "") {
            printf "    {%d} %s https://chooser.crossref.org/?doi=%s ! https://crossmark.crossref.org/dialog?doi=%s\n", count++, entry, doi_number, doi_number;
        } else {
            printf "    {%d} %s\n", count++, entry;
        }
    }
}' "$input_file" > "$output_file"

if [[ ! -s "$output_file" ]]; then
    echo "Error: No data extracted. Please check the input file and patterns."
    exit 1
fi

echo "Data has been extracted to $output_file."