DuckDuckGo URL Collector

Collects URLs from DuckDuckGo with optional site: filtering and rate limiting
安裝腳本 ?
Wrap lines
// ==UserScript==
// @name         DuckDuckGo URL Collector
// @namespace    http://tampermonkey.net/
// @version      1.2.2
// @description  Collects URLs from DuckDuckGo with optional site: filtering and rate limiting
// @author       Ghosty-Tongue
// @match        *://duckduckgo.com/*
// @grant        GM_notification
// @license MIT
// ==/UserScript==
 
(function() {
    'use strict';
 
    const collectedUrls = new Set();
    let isProcessing = false;
    let startTime, timerInterval;
    let targetSite = null;
 
    const searchForm = document.getElementById('search_form_input');
    if (searchForm && searchForm.value.includes('site:')) {
        const match = searchForm.value.match(/site:([^\s]+)/);
        if (match) {
            targetSite = match[1].toLowerCase();
            console.log(`Filtering for site: ${targetSite}`);
        }
    } else {
        console.log('No site: filter detected, collecting all URLs');
    }
 
    const banner = document.createElement('div');
    Object.assign(banner.style, {
        position: 'fixed',
        top: '90px',
        right: '10px',
        zIndex: '10001',
        backgroundColor: 'rgba(255, 165, 0, 0.9)',
        color: 'white',
        padding: '10px',
        borderRadius: '5px',
        display: 'none'
    });
    document.body.appendChild(banner);
 
    const style = document.createElement('style');
    style.textContent = `
        @keyframes rgbFlow {
            0% { background-position: 0% 50%; }
            100% { background-position: 100% 50%; }
        }
        @keyframes pulse {
            0% { transform: scale(1); }
            50% { transform: scale(1.1); }
            100% { transform: scale(1); }
        }
    `;
    document.head.appendChild(style);
 
    const timerDisplay = document.createElement('div');
    Object.assign(timerDisplay.style, {
        position: 'fixed',
        top: '50px',
        right: '10px',
        zIndex: '10000',
        color: 'white',
        backgroundColor: 'rgba(0,0,0,0.7)',
        padding: '5px 10px',
        borderRadius: '5px',
        fontFamily: 'Arial, sans-serif',
        fontSize: '14px'
    });
    document.body.appendChild(timerDisplay);
 
    function startTimer() {
        if (timerInterval) clearInterval(timerInterval);
        startTime = Date.now();
        timerInterval = setInterval(updateTimer, 1000);
        timerDisplay.textContent = '0s';
    }
 
    function updateTimer() {
        const elapsed = Math.floor((Date.now() - startTime) / 1000);
        timerDisplay.textContent = `${elapsed}s`;
    }
 
    function stopTimer() {
        clearInterval(timerInterval);
        const elapsed = Math.floor((Date.now() - startTime) / 1000);
        timerDisplay.textContent = `${elapsed}s (stopped)`;
    }
 
    function extractUrls() {
        console.log('Extracting URLs from current page...');
        const results = document.querySelectorAll('article[data-testid="result"]');
        let newUrlsCount = 0;
 
        results.forEach(result => {
            const link = result.querySelector('a[data-testid="result-extras-url-link"]');
            if (link) {
                const url = link.href;
                const urlDomain = new URL(url).hostname.toLowerCase();
 
                if (targetSite) {
                    if (!urlDomain.includes(targetSite)) {
                        console.log(`Skipping ${url} - doesn't match ${targetSite}`);
                        return;
                    }
                }
 
                if (!collectedUrls.has(url)) {
                    collectedUrls.add(url);
                    newUrlsCount++;
                }
            }
        });
 
        console.log(`Added ${newUrlsCount} new URLs from this batch. Total: ${collectedUrls.size}`);
        return newUrlsCount;
    }
 
    async function clickMoreResults() {
        isProcessing = true;
        btn.classList.add('processing');
        console.log('Starting URL collection process...');
 
        let iteration = 1;
        let moreResultsButton;
        let batchCount = 0;
 
        do {
            if (batchCount >= 420) {
                banner.textContent = 'Taking 15s break to avoid limits';
                banner.style.display = 'block';
                await new Promise(resolve => setTimeout(resolve, 15000));
                banner.style.display = 'none';
                batchCount = 0;
                console.log('Batch counter reset after 420 URLs');
            }
 
            console.log(`Looking for "More Results" button (Attempt ${iteration})...`);
            moreResultsButton = document.getElementById('more-results');
 
            if (moreResultsButton) {
                console.log('Clicking "More Results" button...');
                moreResultsButton.click();
                await new Promise(resolve => setTimeout(resolve, 2000));
                batchCount += extractUrls();
                iteration++;
            }
        } while (moreResultsButton);
 
        console.log(`Finished collecting URLs. Total unique URLs: ${collectedUrls.size}`);
        isProcessing = false;
        btn.classList.remove('processing');
 
        GM_notification({
            title: 'Collection Complete',
            text: `Saved ${collectedUrls.size} URLs`,
            timeout: 5000
        });
 
        saveUrls();
    }
 
    function saveUrls() {
        console.log('Preparing to save URLs to file...');
        const blob = new Blob([Array.from(collectedUrls).join('\n')], {type: 'text/plain'});
        const url = URL.createObjectURL(blob);
        const a = document.createElement('a');
        a.href = url;
        a.download = 'urls.txt';
        document.body.appendChild(a);
        a.click();
        document.body.removeChild(a);
        URL.revokeObjectURL(url);
        console.log(`File saved with ${collectedUrls.size} URLs`);
        stopTimer();
    }
 
    const btn = document.createElement('button');
    btn.textContent = '🦆';
    Object.assign(btn.style, {
        position: 'fixed',
        top: '10px',
        right: '10px',
        zIndex: '10000',
        padding: '12px 24px',
        background: 'linear-gradient(90deg, #ff0000, #00ff00, #0000ff, #ff0000)',
        backgroundSize: '300% 100%',
        animation: 'rgbFlow 5s linear infinite',
        color: 'white',
        border: 'none',
        borderRadius: '25px',
        cursor: 'pointer',
        fontFamily: 'Arial, sans-serif',
        fontWeight: 'bold',
        boxShadow: '0 4px 15px rgba(0,0,0,0.2)',
        transition: 'transform 0.2s, box-shadow 0.2s'
    });
 
    btn.addEventListener('mouseover', () => {
        btn.style.transform = 'scale(1.05)';
        btn.style.boxShadow = '0 6px 20px rgba(0,0,0,0.25)';
    });
 
    btn.addEventListener('mouseout', () => {
        btn.style.transform = 'scale(1)';
        btn.style.boxShadow = '0 4px 15px rgba(0,0,0,0.2)';
    });
 
    btn.addEventListener('click', () => {
        if (!isProcessing) {
            console.log('----- New Collection Started -----');
            collectedUrls.clear();
            startTimer();
            clickMoreResults();
        }
    });
 
    document.body.appendChild(btn);
    console.log('URL Collector button initialized');
})();
欢迎访问Greasy Fork镜像
