From 6f4f37bee53e06f510fd62df13af1c8baaa85861 Mon Sep 17 00:00:00 2001 From: praharshaAdhikari Date: Fri, 31 Oct 2025 23:06:06 +0545 Subject: [PATCH] feat: Instagram scraper with GraphQL API integration - Automated followings list extraction via API interception - Profile scraping using GraphQL endpoint interception - DOM fallback for edge cases - Performance timing for all operations - Anti-bot measures and human-like behavior simulation --- .gitignore | 6 + ANTI-BOT-RECOMMENDATIONS.md | 179 ++++ USAGE-GUIDE.md | 407 +++++++++ package-lock.json | 1648 +++++++++++++++++++++++++++++++++++ package.json | 9 + scraper.js | 723 +++++++++++++++ server.js | 356 ++++++++ utils.js | 146 ++++ 8 files changed, 3474 insertions(+) create mode 100644 ANTI-BOT-RECOMMENDATIONS.md create mode 100644 USAGE-GUIDE.md create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 scraper.js create mode 100644 server.js create mode 100644 utils.js diff --git a/.gitignore b/.gitignore index 2309cc8..92812f2 100644 --- a/.gitignore +++ b/.gitignore @@ -136,3 +136,9 @@ dist .yarn/install-state.gz .pnp.* +# Instagram scraper sensitive files +session_cookies.json +*.json +!package.json +!package-lock.json + diff --git a/ANTI-BOT-RECOMMENDATIONS.md b/ANTI-BOT-RECOMMENDATIONS.md new file mode 100644 index 0000000..c3a04f3 --- /dev/null +++ b/ANTI-BOT-RECOMMENDATIONS.md @@ -0,0 +1,179 @@ +# Instagram Scraper - Anti-Bot Detection Recommendations + +Based on [Scrapfly's Instagram Scraping Guide](https://scrapfly.io/blog/posts/how-to-scrape-instagram) + +## ✅ Already Implemented + +1. **Puppeteer Stealth Plugin** - Bypasses basic browser detection +2. **Random User Agents** - Different browser signatures +3. **Human-like behaviors**: + - Mouse movements + - Random scrolling + - Variable delays (2.5-6 seconds between profiles) + - Typing delays + - Breaks every 10 profiles +4. **Variable viewport sizes** - Randomized window dimensions +5. **Network payload interception** - Capturing API responses instead of DOM scraping +6. **Critical headers** - Including `x-ig-app-id: 936619743392459` + +## ⚠️ Critical Improvements Needed + +### 1. **Residential Proxies** (MOST IMPORTANT) + +**Status**: ❌ Not implemented + +**Issue**: + +- Datacenter IPs (AWS, Google Cloud, etc.) are **blocked instantly** by Instagram +- Your current setup will be detected as soon as you deploy to any cloud server + +**Solution**: + +```javascript +const browser = await puppeteer.launch({ + headless: true, + args: [ + "--proxy-server=residential-proxy-provider.com:port", + // Residential proxies required - NOT datacenter + ], +}); +``` + +**Recommended Proxy Providers**: + +- Bright Data (formerly Luminati) +- Oxylabs +- Smartproxy +- GeoSurf + +**Requirements**: + +- Must be residential IPs (from real ISPs like Comcast, AT&T) +- Rotate IPs every 5-10 minutes (sticky sessions) +- Each IP allows ~200 requests/hour +- Cost: ~$10-15 per GB + +### 2. **Rate Limit Handling with Exponential Backoff** + +**Status**: ⚠️ Partial - needs improvement + +**Current**: Random delays exist +**Needed**: Proper 429 error handling + +```javascript +async function makeRequest(fn, retries = 3) { + for (let i = 0; i < retries; i++) { + try { + return await fn(); + } catch (error) { + if (error.status === 429 && i < retries - 1) { + const delay = Math.pow(2, i) * 2000; // 2s, 4s, 8s + console.log(`Rate limited, waiting ${delay}ms...`); + await new Promise((res) => setTimeout(res, delay)); + continue; + } + throw error; + } + } +} +``` + +### 3. **Session Cookies Management** + +**Status**: ⚠️ Partial - extractSession exists but not reused + +**Issue**: Creating new sessions repeatedly looks suspicious + +**Solution**: + +- Save cookies after login +- Reuse cookies across multiple scraping sessions +- Rotate sessions periodically + +```javascript +// Save cookies after login +const cookies = await extractSession(page); +fs.writeFileSync("session.json", JSON.stringify(cookies)); + +// Reuse cookies in next session +const savedCookies = JSON.parse(fs.readFileSync("session.json")); +await page.setCookie(...savedCookies.cookies); +``` + +### 4. **Realistic Browsing Patterns** + +**Status**: ✅ Implemented but can improve + +**Additional improvements**: + +- Visit homepage before going to target profile +- Occasionally view posts/stories during following list scraping +- Don't always scrape in the same order (randomize) +- Add occasional "browsing breaks" of 30-60 seconds + +### 5. **Monitor doc_id Changes** + +**Status**: ❌ Not monitoring + +**Issue**: Instagram changes GraphQL `doc_id` values every 2-4 weeks + +**Current doc_ids** (as of article): + +- Profile posts: `9310670392322965` +- Post details: `8845758582119845` +- Reels: `25981206651899035` + +**Solution**: + +- Monitor Instagram's GraphQL requests in browser DevTools +- Update when API calls start failing +- Or use a service like Scrapfly that auto-updates + +## 📊 Instagram's Blocking Layers + +1. **IP Quality Check** → Blocks datacenter IPs instantly +2. **TLS Fingerprinting** → Detects non-browser tools (Puppeteer Stealth helps) +3. **Rate Limiting** → ~200 requests/hour per IP +4. **Behavioral Detection** → Flags unnatural patterns + +## 🎯 Priority Implementation Order + +1. **HIGH PRIORITY**: Add residential proxy support +2. **HIGH PRIORITY**: Implement exponential backoff for 429 errors +3. **MEDIUM**: Improve session cookie reuse +4. **MEDIUM**: Add doc_id monitoring system +5. **LOW**: Additional browsing pattern randomization + +## 💰 Cost Estimates (for 10,000 profiles) + +- **Proxy bandwidth**: ~750 MB +- **Cost**: $7.50-$11.25 in residential proxy fees +- **With Proxy Saver**: $5.25-$7.88 (30-50% savings) + +## 🚨 Legal Considerations + +- Only scrape **publicly available** data +- Respect rate limits +- Don't store PII of EU citizens without GDPR compliance +- Add delays to avoid damaging Instagram's servers +- Check Instagram's Terms of Service + +## 📚 Additional Resources + +- [Scrapfly Instagram Scraper](https://github.com/scrapfly/scrapfly-scrapers/tree/main/instagram-scraper) - Open source reference +- [Instagram GraphQL Endpoint Documentation](https://scrapfly.io/blog/posts/how-to-scrape-instagram#how-instagrams-scraping-api-works) +- [Proxy comparison guide](https://scrapfly.io/blog/best-proxy-providers-for-web-scraping) + +## ⚡ Quick Wins + +Things you can implement immediately: + +1. ✅ Critical headers added (x-ig-app-id) +2. ✅ Human simulation functions integrated +3. ✅ Exponential backoff added (see EXPONENTIAL-BACKOFF.md) +4. Implement cookie persistence (15 min) +5. Research residential proxy providers (1 hour) + +--- + +**Bottom Line**: Without residential proxies, this scraper will be blocked immediately on any cloud infrastructure. That's the #1 priority to address. diff --git a/USAGE-GUIDE.md b/USAGE-GUIDE.md new file mode 100644 index 0000000..bca154e --- /dev/null +++ b/USAGE-GUIDE.md @@ -0,0 +1,407 @@ +# Instagram Scraper - Usage Guide + +Complete guide to using the Instagram scraper with all available workflows. + +## 🚀 Quick Start + +### 1. Full Workflow (Recommended) + +The most comprehensive workflow that uses all scraper functions: + +```bash +# Windows PowerShell +$env:INSTAGRAM_USERNAME="your_username" +$env:INSTAGRAM_PASSWORD="your_password" +$env:TARGET_USERNAME="instagram" +$env:MAX_FOLLOWING="20" +$env:MAX_PROFILES="5" +$env:MODE="full" + +node server.js +``` + +**What happens:** + +1. 🔐 **Login** - Logs into Instagram with human-like behavior +2. 💾 **Save Session** - Extracts and saves cookies to `session_cookies.json` +3. 🌐 **Browse** - Simulates random mouse movements and scrolling +4. 👥 **Fetch Followings** - Gets following list using API interception +5. 👤 **Scrape Profiles** - Scrapes detailed data for each profile +6. 📁 **Save Data** - Creates JSON files with all collected data + +**Output files:** + +- `followings_[username]_[timestamp].json` - Full following list +- `profiles_[username]_[timestamp].json` - Detailed profile data +- `session_cookies.json` - Reusable session cookies + +### 2. Simple Workflow + +Uses the built-in `scrapeWorkflow()` function: + +```bash +$env:MODE="simple" +node server.js +``` + +**What it does:** + +- Combines login + following fetch + profile scraping +- Single output file with all data +- Less granular control but simpler + +### 3. Scheduled Workflow + +Runs scraping on a schedule using `cronJobs()`: + +```bash +$env:MODE="scheduled" +$env:SCRAPE_INTERVAL="60" # Minutes between runs +$env:MAX_RUNS="5" # Stop after 5 runs +node server.js +``` + +**Use case:** Monitor a profile's followings over time + +## 📋 Environment Variables + +| Variable | Description | Default | Example | +| -------------------- | ------------------------------------- | --------------- | --------------------- | +| `INSTAGRAM_USERNAME` | Your Instagram username | `your_username` | `john_doe` | +| `INSTAGRAM_PASSWORD` | Your Instagram password | `your_password` | `MySecureP@ss` | +| `TARGET_USERNAME` | Profile to scrape | `instagram` | `cristiano` | +| `MAX_FOLLOWING` | Max followings to fetch | `20` | `100` | +| `MAX_PROFILES` | Max profiles to scrape | `5` | `50` | +| `PROXY` | Proxy server | `None` | `proxy.com:8080` | +| `MODE` | Workflow type | `full` | `simple`, `scheduled` | +| `SCRAPE_INTERVAL` | Minutes between runs (scheduled mode) | `60` | `30` | +| `MAX_RUNS` | Max runs (scheduled mode) | `5` | `10` | + +## 🎯 Workflow Details + +### Full Workflow Step-by-Step + +```javascript +async function fullScrapingWorkflow() { + // Step 1: Login + const { browser, page } = await login(credentials, proxy); + + // Step 2: Extract session + const session = await extractSession(page); + + // Step 3: Simulate browsing + await simulateHumanBehavior(page, { mouseMovements: 5, scrolls: 3 }); + + // Step 4: Get followings list + const followingsData = await getFollowingsList( + page, + targetUsername, + maxFollowing + ); + + // Step 5: Scrape individual profiles + for (const username of followingsData.usernames) { + const profileData = await scrapeProfile(page, username); + // ... takes breaks every 3 profiles + } + + // Step 6: Save all data + // ... creates JSON files +} +``` + +### What Each Function Does + +#### `login(credentials, proxy)` + +- Launches browser with stealth mode +- Sets anti-detection headers +- Simulates human login behavior +- Returns `{ browser, page }` + +#### `extractSession(page)` + +- Gets all cookies from current session +- Returns `{ cookies: [...] }` +- Save for session reuse + +#### `simulateHumanBehavior(page, options)` + +- Random mouse movements +- Random scrolling +- Mimics real user behavior +- Options: `{ mouseMovements, scrolls, randomClicks }` + +#### `getFollowingsList(page, username, maxUsers)` + +- Navigates to profile +- Clicks "following" button +- Intercepts Instagram API responses +- Returns `{ usernames: [...], fullData: [...] }` + +**Full data includes:** + +```json +{ + "pk": "310285748", + "username": "example_user", + "full_name": "Example User", + "profile_pic_url": "https://...", + "is_verified": true, + "is_private": false, + "fbid_v2": "...", + "latest_reel_media": 1761853039 +} +``` + +#### `scrapeProfile(page, username)` + +- Navigates to profile +- Intercepts API endpoint +- Falls back to DOM scraping if needed +- Returns detailed profile data + +**Profile data includes:** + +```json +{ + "username": "example_user", + "full_name": "Example User", + "bio": "Biography text...", + "followerCount": 15000, + "followingCount": 500, + "postsCount": 100, + "is_verified": true, + "is_private": false, + "is_business_account": true, + "email": "contact@example.com", + "phone": "+1234567890" +} +``` + +#### `scrapeWorkflow(creds, targetUsername, proxy, maxFollowing)` + +- Complete workflow in one function +- Combines all steps above +- Returns aggregated results + +#### `cronJobs(fn, intervalSec, stopAfter)` + +- Runs function on interval +- Returns stop function +- Used for scheduled scraping + +## 💡 Usage Examples + +### Example 1: Scrape Top Influencer's Followers + +```bash +$env:INSTAGRAM_USERNAME="your_account" +$env:INSTAGRAM_PASSWORD="your_password" +$env:TARGET_USERNAME="cristiano" +$env:MAX_FOLLOWING="100" +$env:MAX_PROFILES="20" +node server.js +``` + +### Example 2: Monitor Competitor Every Hour + +```bash +$env:TARGET_USERNAME="competitor_account" +$env:MODE="scheduled" +$env:SCRAPE_INTERVAL="60" +$env:MAX_RUNS="24" # Run for 24 hours +node server.js +``` + +### Example 3: Scrape Multiple Accounts + +Create `scrape-multiple.js`: + +```javascript +const { fullScrapingWorkflow } = require("./server.js"); + +const targets = ["account1", "account2", "account3"]; + +async function scrapeAll() { + for (const target of targets) { + process.env.TARGET_USERNAME = target; + await fullScrapingWorkflow(); + + // Wait between accounts + await new Promise((r) => setTimeout(r, 300000)); // 5 minutes + } +} + +scrapeAll(); +``` + +### Example 4: Custom Workflow with Your Logic + +```javascript +const { login, getFollowingsList, scrapeProfile } = require("./scraper.js"); + +async function myCustomWorkflow() { + // Login once + const { browser, page } = await login({ + username: "your_username", + password: "your_password", + }); + + try { + // Get followings from multiple accounts + const accounts = ["account1", "account2"]; + + for (const account of accounts) { + const followings = await getFollowingsList(page, account, 50); + + // Filter verified users only + const verified = followings.fullData.filter((u) => u.is_verified); + + // Scrape verified profiles + for (const user of verified) { + const profile = await scrapeProfile(page, user.username); + + // Custom logic: save only if business account + if (profile.is_business_account) { + console.log(`Business: ${profile.username} - ${profile.email}`); + } + } + } + } finally { + await browser.close(); + } +} + +myCustomWorkflow(); +``` + +## 🔍 Output Format + +### Followings Data + +```json +{ + "targetUsername": "instagram", + "scrapedAt": "2025-10-31T12:00:00.000Z", + "totalFollowings": 20, + "followings": [ + { + "pk": "123456", + "username": "user1", + "full_name": "User One", + "is_verified": true, + ... + } + ] +} +``` + +### Profiles Data + +```json +{ + "targetUsername": "instagram", + "scrapedAt": "2025-10-31T12:00:00.000Z", + "totalProfiles": 5, + "profiles": [ + { + "username": "user1", + "followerCount": 50000, + "email": "contact@user1.com", + ... + } + ] +} +``` + +## ⚡ Performance Tips + +### 1. Optimize Delays + +```javascript +// Faster (more aggressive, higher block risk) +await randomSleep(1000, 2000); + +// Balanced (recommended) +await randomSleep(2500, 6000); + +// Safer (slower but less likely to be blocked) +await randomSleep(5000, 10000); +``` + +### 2. Batch Processing + +Scrape in batches to avoid overwhelming Instagram: + +```javascript +const batchSize = 10; +for (let i = 0; i < usernames.length; i += batchSize) { + const batch = usernames.slice(i, i + batchSize); + // Scrape batch + // Long break between batches + await randomSleep(60000, 120000); // 1-2 minutes +} +``` + +### 3. Session Reuse + +Reuse cookies to avoid logging in repeatedly: + +```javascript +const savedCookies = JSON.parse(fs.readFileSync("session_cookies.json")); +await page.setCookie(...savedCookies.cookies); +``` + +## 🚨 Common Issues + +### "Rate limited (429)" + +✅ **Solution**: Exponential backoff is automatic. If persistent: + +- Reduce MAX_FOLLOWING and MAX_PROFILES +- Increase delays +- Add residential proxies + +### "Login failed" + +- Check credentials +- Instagram may require verification +- Try from your home IP first + +### "No data captured" + +- Instagram changed their API structure +- Check if `doc_id` values need updating +- DOM fallback should still work + +### Blocked on cloud servers + +❌ **Problem**: Using datacenter IPs +✅ **Solution**: Get residential proxies (see ANTI-BOT-RECOMMENDATIONS.md) + +## 📊 Best Practices + +1. **Start Small**: Test with MAX_FOLLOWING=5, MAX_PROFILES=2 +2. **Use Residential Proxies**: Critical for production use +3. **Respect Rate Limits**: ~200 requests/hour per IP +4. **Save Sessions**: Reuse cookies to avoid repeated logins +5. **Monitor Logs**: Watch for 429 errors +6. **Add Randomness**: Vary delays and patterns +7. **Take Breaks**: Schedule longer breaks every N profiles + +## 🎓 Learning Path + +1. **Start**: Run `MODE=simple` with small numbers +2. **Understand**: Read the logs and output files +3. **Customize**: Modify `MAX_FOLLOWING` and `MAX_PROFILES` +4. **Advanced**: Use `MODE=full` for complete control +5. **Production**: Add proxies and session management + +--- + +**Need help?** Check: + +- [ANTI-BOT-RECOMMENDATIONS.md](./ANTI-BOT-RECOMMENDATIONS.md) +- [EXPONENTIAL-BACKOFF.md](./EXPONENTIAL-BACKOFF.md) +- Test script: `node test-retry.js` diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..fec8fa1 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,1648 @@ +{ + "name": "instagram-scraper", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "dotenv": "^17.2.3", + "puppeteer": "^24.27.0", + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-stealth": "^2.11.2", + "random-useragent": "^0.5.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@puppeteer/browsers": { + "version": "2.10.12", + "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.10.12.tgz", + "integrity": "sha512-mP9iLFZwH+FapKJLeA7/fLqOlSUwYpMwjR1P5J23qd4e7qGJwecJccJqHYrjw33jmIZYV4dtiTHPD/J+1e7cEw==", + "license": "Apache-2.0", + "dependencies": { + "debug": "^4.4.3", + "extract-zip": "^2.0.1", + "progress": "^2.0.3", + "proxy-agent": "^6.5.0", + "semver": "^7.7.3", + "tar-fs": "^3.1.1", + "yargs": "^17.7.2" + }, + "bin": { + "browsers": "lib/cjs/main-cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@tootallnate/quickjs-emscripten": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@tootallnate/quickjs-emscripten/-/quickjs-emscripten-0.23.0.tgz", + "integrity": "sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==", + "license": "MIT" + }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.9.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.9.2.tgz", + "integrity": "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA==", + "license": "MIT", + "optional": true, + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/yauzl": { + "version": "2.10.3", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.10.3.tgz", + "integrity": "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==", + "license": "MIT", + "optional": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/arr-union": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", + "integrity": "sha512-sKpyeERZ02v1FeCZT8lrfJq5u6goHCtpTAzPwJYe7c8SPFOboNjNg1vz2L4VTn9T4PQxEx13TbXLmYUcS6Ug7Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ast-types": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/ast-types/-/ast-types-0.13.4.tgz", + "integrity": "sha512-x1FCFnFifvYDDzTaLII71vG5uvDwgtmDTEVWAxrgeiR8VjMONcCXJx7E+USjDtHlwFmt9MysbqgF9b9Vjr6w+w==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/b4a": { + "version": "1.7.3", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.7.3.tgz", + "integrity": "sha512-5Q2mfq2WfGuFp3uS//0s6baOJLMoVduPYVeNmDYxu5OUA1/cBfvr2RIS7vi62LdNj/urk1hfmj867I3qt6uZ7Q==", + "license": "Apache-2.0", + "peerDependencies": { + "react-native-b4a": "*" + }, + "peerDependenciesMeta": { + "react-native-b4a": { + "optional": true + } + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "license": "MIT" + }, + "node_modules/bare-events": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.8.1.tgz", + "integrity": "sha512-oxSAxTS1hRfnyit2CL5QpAOS5ixfBjj6ex3yTNvXyY/kE719jQ/IjuESJBK2w5v4wwQRAHGseVJXx9QBYOtFGQ==", + "license": "Apache-2.0", + "peerDependencies": { + "bare-abort-controller": "*" + }, + "peerDependenciesMeta": { + "bare-abort-controller": { + "optional": true + } + } + }, + "node_modules/bare-fs": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.5.0.tgz", + "integrity": "sha512-GljgCjeupKZJNetTqxKaQArLK10vpmK28or0+RwWjEl5Rk+/xG3wkpmkv+WrcBm3q1BwHKlnhXzR8O37kcvkXQ==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-events": "^2.5.4", + "bare-path": "^3.0.0", + "bare-stream": "^2.6.4", + "bare-url": "^2.2.2", + "fast-fifo": "^1.3.2" + }, + "engines": { + "bare": ">=1.16.0" + }, + "peerDependencies": { + "bare-buffer": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + } + } + }, + "node_modules/bare-os": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.6.2.tgz", + "integrity": "sha512-T+V1+1srU2qYNBmJCXZkUY5vQ0B4FSlL3QDROnKQYOqeiQR8UbjNHlPa+TIbM4cuidiN9GaTaOZgSEgsvPbh5A==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "bare": ">=1.14.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.7.0.tgz", + "integrity": "sha512-oyXQNicV1y8nc2aKffH+BUHFRXmx6VrPzlnaEvMhram0nPBrKcEdcyBg5r08D0i8VxngHFAiVyn1QKXpSG0B8A==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.21.0" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/bare-url": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/bare-url/-/bare-url-2.3.1.tgz", + "integrity": "sha512-v2yl0TnaZTdEnelkKtXZGnotiV6qATBlnNuUMrHl6v9Lmmrh9mw9RYyImPU7/4RahumSwQS1k2oKXcRfXcbjJw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-path": "^3.0.0" + } + }, + "node_modules/basic-ftp": { + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.0.5.tgz", + "integrity": "sha512-4Bcg1P8xhUuqcii/S0Z9wiHIrQVPMermM1any+MX5GeGD7faD3/msQUDGLol9wOcz4/jbg/WJnGqoJF6LiBdtg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/chromium-bidi": { + "version": "10.5.1", + "resolved": "https://registry.npmjs.org/chromium-bidi/-/chromium-bidi-10.5.1.tgz", + "integrity": "sha512-rlj6OyhKhVTnk4aENcUme3Jl9h+cq4oXu4AzBcvr8RMmT6BR4a3zSNT9dbIfXr9/BS6ibzRyDhowuw4n2GgzsQ==", + "license": "Apache-2.0", + "dependencies": { + "mitt": "^3.0.1", + "zod": "^3.24.1" + }, + "peerDependencies": { + "devtools-protocol": "*" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/clone-deep": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-0.2.4.tgz", + "integrity": "sha512-we+NuQo2DHhSl+DP6jlUiAhyAjBQrYnpOk15rN6c6JSPScjiCLh8IbSU+VTcph6YS3o7mASE8a0+gbZ7ChLpgg==", + "license": "MIT", + "dependencies": { + "for-own": "^0.1.3", + "is-plain-object": "^2.0.1", + "kind-of": "^3.0.2", + "lazy-cache": "^1.0.3", + "shallow-clone": "^0.1.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "license": "MIT" + }, + "node_modules/cosmiconfig": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-9.0.0.tgz", + "integrity": "sha512-itvL5h8RETACmOTFc4UfIyB2RfEHi71Ax6E/PivVxq9NseKbOWpeyHEOIbmAw1rs8Ak0VursQNww7lf7YtUwzg==", + "license": "MIT", + "dependencies": { + "env-paths": "^2.2.1", + "import-fresh": "^3.3.0", + "js-yaml": "^4.1.0", + "parse-json": "^5.2.0" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/d-fischer" + }, + "peerDependencies": { + "typescript": ">=4.9.5" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/data-uri-to-buffer": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-6.0.2.tgz", + "integrity": "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/degenerator": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", + "integrity": "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ==", + "license": "MIT", + "dependencies": { + "ast-types": "^0.13.4", + "escodegen": "^2.1.0", + "esprima": "^4.0.1" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/devtools-protocol": { + "version": "0.0.1521046", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1521046.tgz", + "integrity": "sha512-vhE6eymDQSKWUXwwA37NtTTVEzjtGVfDr3pRbsWEQ5onH/Snp2c+2xZHWJJawG/0hCCJLRGt4xVtEVUVILol4w==", + "license": "BSD-3-Clause", + "peer": true + }, + "node_modules/dotenv": { + "version": "17.2.3", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", + "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/env-paths": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", + "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/error-ex": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escodegen": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.1.0.tgz", + "integrity": "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w==", + "license": "BSD-2-Clause", + "dependencies": { + "esprima": "^4.0.1", + "estraverse": "^5.2.0", + "esutils": "^2.0.2" + }, + "bin": { + "escodegen": "bin/escodegen.js", + "esgenerate": "bin/esgenerate.js" + }, + "engines": { + "node": ">=6.0" + }, + "optionalDependencies": { + "source-map": "~0.6.1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/estraverse": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/events-universal": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/events-universal/-/events-universal-1.0.1.tgz", + "integrity": "sha512-LUd5euvbMLpwOF8m6ivPCbhQeSiYVNb8Vs0fQ8QjXo0JTkEHpz8pxdQf0gStltaPpw0Cca8b39KxvK9cfKRiAw==", + "license": "Apache-2.0", + "dependencies": { + "bare-events": "^2.7.0" + } + }, + "node_modules/extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "license": "BSD-2-Clause", + "dependencies": { + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "bin": { + "extract-zip": "cli.js" + }, + "engines": { + "node": ">= 10.17.0" + }, + "optionalDependencies": { + "@types/yauzl": "^2.9.1" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", + "license": "MIT", + "dependencies": { + "pend": "~1.2.0" + } + }, + "node_modules/for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha512-7EwmXrOjyL+ChxMhmG5lnW9MPt1aIeZEwKhQzoBUdTV0N3zuwWDZYVJatDvZ2OyzPUvdIAZDsCetk3coyMfcnQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/for-own": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/for-own/-/for-own-0.1.5.tgz", + "integrity": "sha512-SKmowqGTJoPzLO1T0BBJpkfp3EMacCMOuH40hOUbrbzElVktk4DioXVM99QkLCyKoiuOmyjgcWMpVz2xjE7LZw==", + "license": "MIT", + "dependencies": { + "for-in": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fs-extra": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "license": "ISC" + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-uri": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/get-uri/-/get-uri-6.0.5.tgz", + "integrity": "sha512-b1O07XYq8eRuVzBNgJLstU6FYc1tS6wnMtF1I1D9lE8LxZSOGZ7LhxN54yPP6mGw5f2CkXY2BQUL9Fx41qvcIg==", + "license": "MIT", + "dependencies": { + "basic-ftp": "^5.0.2", + "data-uri-to-buffer": "^6.0.2", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ip-address": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.0.1.tgz", + "integrity": "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "license": "MIT" + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "license": "MIT", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "license": "ISC" + }, + "node_modules/jsonfile": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.0.tgz", + "integrity": "sha512-FGuPw30AdOIUTRMC2OMRtQV+jkVj2cfPqSeWXv1NEAJ1qZ5zb1X6z1mFhbfOB/iy3ssJCD+3KuZ8r8C3uVFlAg==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==", + "license": "MIT", + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lazy-cache": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", + "integrity": "sha512-RE2g0b5VGZsOCFOCgP7omTRYFqydmZkBwl5oNnQ1lDYC57uyO9KqNnNVxT7COSHTxrRCWVcAVOcbjk+tvh/rgQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "7.18.3", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-7.18.3.tgz", + "integrity": "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/merge-deep": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/merge-deep/-/merge-deep-3.0.3.tgz", + "integrity": "sha512-qtmzAS6t6grwEkNrunqTBdn0qKwFgNWvlxUbAV8es9M7Ot1EbyApytCnvE0jALPa46ZpKDUo527kKiaWplmlFA==", + "license": "MIT", + "dependencies": { + "arr-union": "^3.1.0", + "clone-deep": "^0.2.4", + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, + "node_modules/mixin-object": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mixin-object/-/mixin-object-2.0.1.tgz", + "integrity": "sha512-ALGF1Jt9ouehcaXaHhn6t1yGWRqGaHkPFndtFVHfZXOvkIZ/yoGaSi0AHVTafb3ZBGg4dr/bDwnaEKqCXzchMA==", + "license": "MIT", + "dependencies": { + "for-in": "^0.1.3", + "is-extendable": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mixin-object/node_modules/for-in": { + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-0.1.8.tgz", + "integrity": "sha512-F0to7vbBSHP8E3l6dCjxNOLuSFAACIxFy3UehTUlG7svlXi37HHsDkyVcHo0Pq8QwrE+pXvWSVX3ZT1T9wAZ9g==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/netmask": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", + "integrity": "sha512-dBpDMdxv9Irdq66304OLfEmQ9tbNRFnFTuZiLo+bD+r332bBmMJ8GBLXklIXXgxd3+v9+KUnZaUR5PJMa75Gsg==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/pac-proxy-agent": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/pac-proxy-agent/-/pac-proxy-agent-7.2.0.tgz", + "integrity": "sha512-TEB8ESquiLMc0lV8vcd5Ql/JAKAoyzHFXaStwjkzpOpC5Yv+pIzLfHvjTSdf3vpa2bMiUQrg9i6276yn8666aA==", + "license": "MIT", + "dependencies": { + "@tootallnate/quickjs-emscripten": "^0.23.0", + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "get-uri": "^6.0.1", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.6", + "pac-resolver": "^7.0.1", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/pac-resolver": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pac-resolver/-/pac-resolver-7.0.1.tgz", + "integrity": "sha512-5NPgf87AT2STgwa2ntRMr45jTKrYBGkVU36yT0ig/n/GMAa3oPqhZfIQ2kMEimReg0+t9kZViDVZ83qfVUlckg==", + "license": "MIT", + "dependencies": { + "degenerator": "^5.0.0", + "netmask": "^2.0.2" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==", + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/proxy-agent": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/proxy-agent/-/proxy-agent-6.5.0.tgz", + "integrity": "sha512-TmatMXdr2KlRiA2CyDu8GqR8EjahTG3aY3nXjdzFyoZbmB8hrBsTyMezhULIXKnC0jpfjlmiZ3+EaCzoInSu/A==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "http-proxy-agent": "^7.0.1", + "https-proxy-agent": "^7.0.6", + "lru-cache": "^7.14.1", + "pac-proxy-agent": "^7.1.0", + "proxy-from-env": "^1.1.0", + "socks-proxy-agent": "^8.0.5" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, + "node_modules/pump": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", + "integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/puppeteer": { + "version": "24.27.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-24.27.0.tgz", + "integrity": "sha512-eEcAFGxmHRSrk74DVkFAMAwfj4l3Ak8avBuA2bZaAoocY1+Fb9WLS3I7jlOc/tIOU7EmGLiDdVP08R44wADpHw==", + "hasInstallScript": true, + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@puppeteer/browsers": "2.10.12", + "chromium-bidi": "10.5.1", + "cosmiconfig": "^9.0.0", + "devtools-protocol": "0.0.1521046", + "puppeteer-core": "24.27.0", + "typed-query-selector": "^2.12.0" + }, + "bin": { + "puppeteer": "lib/cjs/puppeteer/node/cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-core": { + "version": "24.27.0", + "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-24.27.0.tgz", + "integrity": "sha512-yubwj2XXmTM3wRIpbhO5nCjbByPgpFHlgrsD4IK+gMPqO7/a5FfnoSXDKjmqi8A2M1Ewusz0rTI/r+IN0GU0MA==", + "license": "Apache-2.0", + "dependencies": { + "@puppeteer/browsers": "2.10.12", + "chromium-bidi": "10.5.1", + "debug": "^4.4.3", + "devtools-protocol": "0.0.1521046", + "typed-query-selector": "^2.12.0", + "webdriver-bidi-protocol": "0.3.8", + "ws": "^8.18.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/puppeteer-extra": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/puppeteer-extra/-/puppeteer-extra-3.3.6.tgz", + "integrity": "sha512-rsLBE/6mMxAjlLd06LuGacrukP2bqbzKCLzV1vrhHFavqQE/taQ2UXv3H5P0Ls7nsrASa+6x3bDbXHpqMwq+7A==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/debug": "^4.1.0", + "debug": "^4.1.1", + "deepmerge": "^4.2.2" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "@types/puppeteer": "*", + "puppeteer": "*", + "puppeteer-core": "*" + }, + "peerDependenciesMeta": { + "@types/puppeteer": { + "optional": true + }, + "puppeteer": { + "optional": true + }, + "puppeteer-core": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin/-/puppeteer-extra-plugin-3.2.3.tgz", + "integrity": "sha512-6RNy0e6pH8vaS3akPIKGg28xcryKscczt4wIl0ePciZENGE2yoaQJNd17UiEbdmh5/6WW6dPcfRWT9lxBwCi2Q==", + "license": "MIT", + "dependencies": { + "@types/debug": "^4.1.0", + "debug": "^4.1.1", + "merge-deep": "^3.0.1" + }, + "engines": { + "node": ">=9.11.2" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-stealth": { + "version": "2.11.2", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.11.2.tgz", + "integrity": "sha512-bUemM5XmTj9i2ZerBzsk2AN5is0wHMNE6K0hXBzBXOzP5m5G3Wl0RHhiqKeHToe/uIH8AoZiGhc1tCkLZQPKTQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.1", + "puppeteer-extra-plugin": "^3.2.3", + "puppeteer-extra-plugin-user-preferences": "^2.4.1" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-user-data-dir": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-data-dir/-/puppeteer-extra-plugin-user-data-dir-2.4.1.tgz", + "integrity": "sha512-kH1GnCcqEDoBXO7epAse4TBPJh9tEpVEK/vkedKfjOVOhZAvLkHGc9swMs5ChrJbRnf8Hdpug6TJlEuimXNQ+g==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.1", + "fs-extra": "^10.0.0", + "puppeteer-extra-plugin": "^3.2.3", + "rimraf": "^3.0.2" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/puppeteer-extra-plugin-user-preferences": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-user-preferences/-/puppeteer-extra-plugin-user-preferences-2.4.1.tgz", + "integrity": "sha512-i1oAZxRbc1bk8MZufKCruCEC3CCafO9RKMkkodZltI4OqibLFXF3tj6HZ4LZ9C5vCXZjYcDWazgtY69mnmrQ9A==", + "license": "MIT", + "dependencies": { + "debug": "^4.1.1", + "deepmerge": "^4.2.2", + "puppeteer-extra-plugin": "^3.2.3", + "puppeteer-extra-plugin-user-data-dir": "^2.4.1" + }, + "engines": { + "node": ">=8" + }, + "peerDependencies": { + "playwright-extra": "*", + "puppeteer-extra": "*" + }, + "peerDependenciesMeta": { + "playwright-extra": { + "optional": true + }, + "puppeteer-extra": { + "optional": true + } + } + }, + "node_modules/random-seed": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/random-seed/-/random-seed-0.3.0.tgz", + "integrity": "sha512-y13xtn3kcTlLub3HKWXxJNeC2qK4mB59evwZ5EkeRlolx+Bp2ztF7LbcZmyCnOqlHQrLnfuNbi1sVmm9lPDlDA==", + "license": "MIT", + "dependencies": { + "json-stringify-safe": "^5.0.1" + }, + "engines": { + "node": ">= 0.6.0" + } + }, + "node_modules/random-useragent": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/random-useragent/-/random-useragent-0.5.0.tgz", + "integrity": "sha512-FUMkqVdZeoSff5tErNL3FFGYXElDWZ1bEuedhm5u9MdCFwANriJWbHvDRYrLTOzp/fBsBGu5J1cWtDgifa97aQ==", + "license": "MIT", + "dependencies": { + "random-seed": "^0.3.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shallow-clone": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-0.1.2.tgz", + "integrity": "sha512-J1zdXCky5GmNnuauESROVu31MQSnLoYvlyEn6j2Ztk6Q5EHFIhxkMhYcv6vuDzl2XEzoRr856QwzMgWM/TmZgw==", + "license": "MIT", + "dependencies": { + "is-extendable": "^0.1.1", + "kind-of": "^2.0.1", + "lazy-cache": "^0.2.3", + "mixin-object": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shallow-clone/node_modules/kind-of": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-2.0.1.tgz", + "integrity": "sha512-0u8i1NZ/mg0b+W3MGGw5I7+6Eib2nx72S/QvXa0hYjEkjTknYmEYQJwGu3mLC0BrhtJjtQafTkyRUQ75Kx0LVg==", + "license": "MIT", + "dependencies": { + "is-buffer": "^1.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shallow-clone/node_modules/lazy-cache": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-0.2.7.tgz", + "integrity": "sha512-gkX52wvU/R8DVMMt78ATVPFMJqfW8FPz1GZ1sVHBVQHmu/WvhIWE4cE1GBzhJNFicDeYhnwp6Rl35BcAIM3YOQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "license": "MIT", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks": { + "version": "2.8.7", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.8.7.tgz", + "integrity": "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A==", + "license": "MIT", + "dependencies": { + "ip-address": "^10.0.1", + "smart-buffer": "^4.2.0" + }, + "engines": { + "node": ">= 10.0.0", + "npm": ">= 3.0.0" + } + }, + "node_modules/socks-proxy-agent": { + "version": "8.0.5", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-8.0.5.tgz", + "integrity": "sha512-HehCEsotFqbPW9sJ8WVYB6UbmIMv7kUUORIF2Nncq4VQvBfNBLibW9YZR5dlYCSUhwcD628pRllm7n+E+YTzJw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "^4.3.4", + "socks": "^2.8.3" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "optional": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/streamx": { + "version": "2.23.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.23.0.tgz", + "integrity": "sha512-kn+e44esVfn2Fa/O0CPFcex27fjIL6MkVae0Mm6q+E6f0hWv578YCERbv+4m02cjxvDsPKLnmxral/rR6lBMAg==", + "license": "MIT", + "dependencies": { + "events-universal": "^1.0.0", + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tar-fs": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz", + "integrity": "sha512-LZA0oaPOc2fVo82Txf3gw+AkEd38szODlptMYejQUhndHMLQ9M059uXR+AfS7DNo0NpINvSqDsvyaCrBVkptWg==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typed-query-selector": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/typed-query-selector/-/typed-query-selector-2.12.0.tgz", + "integrity": "sha512-SbklCd1F0EiZOyPiW192rrHZzZ5sBijB6xM+cpmrwDqObvdtunOHHIk9fCGsoK5JVIYXoyEp4iEdE3upFH3PAg==", + "license": "MIT" + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "license": "MIT", + "optional": true + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/webdriver-bidi-protocol": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/webdriver-bidi-protocol/-/webdriver-bidi-protocol-0.3.8.tgz", + "integrity": "sha512-21Yi2GhGntMc671vNBCjiAeEVknXjVRoyu+k+9xOMShu+ZQfpGQwnBqbNz/Sv4GXZ6JmutlPAi2nIJcrymAWuQ==", + "license": "Apache-2.0" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.18.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==", + "license": "MIT", + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..e4b2110 --- /dev/null +++ b/package.json @@ -0,0 +1,9 @@ +{ + "dependencies": { + "dotenv": "^17.2.3", + "puppeteer": "^24.27.0", + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-stealth": "^2.11.2", + "random-useragent": "^0.5.0" + } +} diff --git a/scraper.js b/scraper.js new file mode 100644 index 0000000..1b1aa02 --- /dev/null +++ b/scraper.js @@ -0,0 +1,723 @@ +const puppeteer = require("puppeteer-extra"); +const StealthPlugin = require("puppeteer-extra-plugin-stealth"); +const randomUseragent = require("random-useragent"); +const fs = require("fs"); +const { + randomSleep, + simulateHumanBehavior, + handleRateLimitedRequest, +} = require("./utils.js"); + +puppeteer.use(StealthPlugin()); + +const INSTAGRAM_URL = "https://www.instagram.com"; +const SESSION_FILE = "session_cookies.json"; + +async function loginWithSession( + { username, password }, + proxy = null, + useExistingSession = true +) { + const browserArgs = []; + if (proxy) browserArgs.push(`--proxy-server=${proxy}`); + const userAgent = randomUseragent.getRandom(); + + const browser = await puppeteer.launch({ + headless: false, + args: browserArgs, + }); + const page = await browser.newPage(); + await page.setUserAgent(userAgent); + + // Set a large viewport to ensure modal behavior (Instagram shows modals on desktop/large screens) + await page.setViewport({ + width: 1920, // Standard desktop width + height: 1080, // Standard desktop height + }); + + // Set browser timezone + await page.evaluateOnNewDocument(() => { + Object.defineProperty(Intl.DateTimeFormat.prototype, "resolvedOptions", { + value: function () { + return { timeZone: "America/New_York" }; + }, + }); + }); + + // Monitor for rate limit responses + page.on("response", (response) => { + if (response.status() === 429) { + console.log( + `WARNING: Rate limit detected (429) on ${response + .url() + .substring(0, 80)}...` + ); + } + }); + + // Try to load existing session if available + if (useExistingSession && fs.existsSync(SESSION_FILE)) { + try { + console.log("Found existing session, attempting to reuse..."); + const sessionData = JSON.parse(fs.readFileSync(SESSION_FILE, "utf-8")); + + if (sessionData.cookies && sessionData.cookies.length > 0) { + await page.setCookie(...sessionData.cookies); + console.log( + `Loaded ${sessionData.cookies.length} cookies from session` + ); + + // Navigate to Instagram to check if session is valid + await page.goto(INSTAGRAM_URL, { waitUntil: "networkidle2" }); + await randomSleep(2000, 3000); + + // Check if we're logged in by looking for profile link or login page + const isLoggedIn = await page.evaluate(() => { + // If we see login/signup links, we're not logged in + const loginLink = document.querySelector( + 'a[href="/accounts/login/"]' + ); + return !loginLink; + }); + + if (isLoggedIn) { + console.log("Session is valid! Skipping login."); + return { browser, page, sessionReused: true }; + } else { + console.log("Session expired, proceeding with fresh login..."); + } + } + } catch (error) { + console.log("Failed to load session, proceeding with fresh login..."); + } + } + + // Fresh login flow + return await performLogin(page, { username, password }, browser); +} + +async function performLogin(page, { username, password }, browser) { + // Navigate to login page + await handleRateLimitedRequest( + page, + async () => { + await page.goto(`${INSTAGRAM_URL}/accounts/login/`, { + waitUntil: "networkidle2", + }); + }, + "during login page load" + ); + + console.log("Waiting for login form to appear..."); + + // Wait for the actual login form to load + await page.waitForSelector('input[name="username"]', { + visible: true, + timeout: 60000, + }); + + console.log("Login form loaded!"); + + // Simulate human behavior + await simulateHumanBehavior(page, { mouseMovements: 3, scrolls: 1 }); + await randomSleep(500, 1000); + + await page.type('input[name="username"]', username, { delay: 130 }); + await randomSleep(300, 700); + await page.type('input[name="password"]', password, { delay: 120 }); + + await simulateHumanBehavior(page, { mouseMovements: 2, scrolls: 0 }); + await randomSleep(500, 1000); + + await Promise.all([ + page.click('button[type="submit"]'), + page.waitForNavigation({ waitUntil: "networkidle2" }), + ]); + + await randomSleep(1000, 2000); + + return { browser, page, sessionReused: false }; +} + +async function extractSession(page) { + // Return cookies/session tokens for reuse + const cookies = await page.cookies(); + return { cookies }; +} + +async function getFollowingsList(page, targetUsername, maxUsers = 100) { + const followingData = []; + const followingUsernames = []; + let requestCount = 0; + const requestsPerBatch = 12; // Instagram typically returns ~12 users per request + + // Set up response listener to capture API responses (no need for request interception) + page.on("response", async (response) => { + const url = response.url(); + + // Intercept the following list API endpoint + if (url.includes("/friendships/") && url.includes("/following/")) { + try { + const json = await response.json(); + + // Check for rate limit in response + if (json.status === "fail" || json.message?.includes("rate limit")) { + console.log("WARNING: Rate limit detected in API response"); + return; + } + + if (json.users && Array.isArray(json.users)) { + json.users.forEach((user) => { + if (followingData.length < maxUsers) { + followingData.push({ + pk: user.pk, + pk_id: user.pk_id, + username: user.username, + full_name: user.full_name, + profile_pic_url: user.profile_pic_url, + is_verified: user.is_verified, + is_private: user.is_private, + fbid_v2: user.fbid_v2, + latest_reel_media: user.latest_reel_media, + account_badges: user.account_badges, + }); + followingUsernames.push(user.username); + } + }); + + requestCount++; + console.log( + `Captured ${followingData.length} users so far (Request #${requestCount})...` + ); + } + } catch (err) { + // Not JSON or parsing error, ignore + } + } + }); + + await handleRateLimitedRequest( + page, + async () => { + await page.goto(`${INSTAGRAM_URL}/${targetUsername}/`, { + waitUntil: "networkidle2", + }); + }, + `while loading profile @${targetUsername}` + ); + + // Simulate browsing the profile before clicking following + await simulateHumanBehavior(page, { mouseMovements: 4, scrolls: 2 }); + await randomSleep(1000, 2000); + + await page.waitForSelector('a[href$="/following/"]', { timeout: 10000 }); + + // Hover over the following link before clicking + await page.hover('a[href$="/following/"]'); + await randomSleep(300, 600); + + await page.click('a[href$="/following/"]'); + + // Wait for either modal or page navigation + await randomSleep(1500, 2500); + + // Detect if modal opened or if we navigated to a new page + const layoutType = await page.evaluate(() => { + const hasModal = !!document.querySelector('div[role="dialog"]'); + const urlHasFollowing = window.location.pathname.includes("/following"); + return { hasModal, urlHasFollowing }; + }); + + if (layoutType.hasModal) { + console.log("Following modal opened (desktop layout)"); + } else if (layoutType.urlHasFollowing) { + console.log("Navigated to following page (mobile/small viewport layout)"); + } else { + console.log("Warning: Could not detect following list layout"); + } + + // Wait for the list content to load + await randomSleep(1500, 2500); + + // Verify we can see the list items + const hasListItems = await page.evaluate(() => { + return ( + document.querySelectorAll('div.x1qnrgzn, a[href*="following"]').length > 0 + ); + }); + + if (hasListItems) { + console.log("Following list loaded successfully"); + } else { + console.log("Warning: List items not detected, but continuing..."); + } + + // Scroll to load more users while simulating human behavior + const totalRequests = Math.ceil(maxUsers / requestsPerBatch); + let scrollAttempts = 0; + const maxScrollAttempts = Math.min(totalRequests * 3, 50000); // Cap at 50k attempts + let lastDataLength = 0; + let noNewDataCount = 0; + + console.log( + `Will attempt to scroll up to ${maxScrollAttempts} times to reach ${maxUsers} users...` + ); + + while ( + followingData.length < maxUsers && + scrollAttempts < maxScrollAttempts + ) { + // Check if we're still getting new data + if (followingData.length === lastDataLength) { + noNewDataCount++; + // If no new data after 8 consecutive scroll attempts, we've reached the end + if (noNewDataCount >= 8) { + console.log( + `No new data after ${noNewDataCount} attempts. Reached end of list.` + ); + break; + } + if (noNewDataCount % 3 === 0) { + console.log( + `Still at ${followingData.length} users after ${noNewDataCount} scrolls...` + ); + } + } else { + if (noNewDataCount > 0) { + console.log( + `Got new data! Now at ${followingData.length} users (was stuck for ${noNewDataCount} attempts)` + ); + } + noNewDataCount = 0; // Reset counter when we get new data + lastDataLength = followingData.length; + } + + // Every ~12 users loaded (one request completed), simulate human behavior + if ( + requestCount > 0 && + requestCount % Math.max(1, Math.ceil(totalRequests / 5)) === 0 + ) { + await simulateHumanBehavior(page, { + mouseMovements: 2, + scrolls: 0, // We're manually controlling scroll below + }); + } + + // Occasionally move mouse while scrolling + if (scrollAttempts % 5 === 0) { + const viewport = await page.viewport(); + await page.mouse.move( + Math.floor(Math.random() * viewport.width), + Math.floor(Math.random() * viewport.height), + { steps: 10 } + ); + } + + // Scroll the dialog's scrollable container - comprehensive approach + const scrollResult = await page.evaluate(() => { + // Find the scrollable container inside the dialog + const dialog = document.querySelector('div[role="dialog"]'); + if (!dialog) { + return { success: false, error: "No dialog found", scrolled: false }; + } + + // Look for the scrollable div - it has overflow: hidden auto + const scrollableElements = dialog.querySelectorAll("div"); + let scrollContainer = null; + + for (const elem of scrollableElements) { + const style = window.getComputedStyle(elem); + const overflow = style.overflow || style.overflowY; + + // Check if element is scrollable + if ( + (overflow === "auto" || overflow === "scroll") && + elem.scrollHeight > elem.clientHeight + ) { + scrollContainer = elem; + break; + } + } + + if (!scrollContainer) { + // Fallback: try specific class from your HTML + scrollContainer = + dialog.querySelector("div.x6nl9eh") || + dialog.querySelector('div[style*="overflow"]'); + } + + if (!scrollContainer) { + return { + success: false, + error: "No scrollable container found", + scrolled: false, + }; + } + + const oldScrollTop = scrollContainer.scrollTop; + const scrollHeight = scrollContainer.scrollHeight; + const clientHeight = scrollContainer.clientHeight; + + // Scroll down + scrollContainer.scrollTop += 400 + Math.floor(Math.random() * 200); + + const newScrollTop = scrollContainer.scrollTop; + const actuallyScrolled = newScrollTop > oldScrollTop; + const atBottom = scrollHeight - newScrollTop - clientHeight < 50; + + return { + success: true, + scrolled: actuallyScrolled, + atBottom: atBottom, + scrollTop: newScrollTop, + scrollHeight: scrollHeight, + }; + }); + + if (!scrollResult.success) { + console.log(`Scroll error: ${scrollResult.error}`); + // Try alternative: scroll the page itself + await page.evaluate(() => window.scrollBy(0, 300)); + } else if (!scrollResult.scrolled) { + console.log("Reached scroll bottom - cannot scroll further"); + } + + // Check if we've reached the bottom and loading indicator is visible + const loadingStatus = await page.evaluate(() => { + const loader = document.querySelector('svg[aria-label="Loading..."]'); + + if (!loader) { + return { exists: false, visible: false, reachedBottom: true }; + } + + // Check if loader is in viewport (visible) + const rect = loader.getBoundingClientRect(); + const isVisible = + rect.top >= 0 && + rect.left >= 0 && + rect.bottom <= window.innerHeight && + rect.right <= window.innerWidth; + + return { exists: true, visible: isVisible, reachedBottom: isVisible }; + }); + + if (!loadingStatus.exists) { + // No loading indicator at all - might have reached the actual end + console.log("No loading indicator found - may have reached end of list"); + } else if (loadingStatus.visible) { + // Loader is visible, meaning we've scrolled to it + console.log("Loading indicator visible, waiting for more data..."); + await randomSleep(2500, 3500); // Wait longer for Instagram to load more + } else { + // Loader exists but not visible yet, keep scrolling + await randomSleep(1500, 2500); + } + + scrollAttempts++; + + // Progress update every 50 scrolls + if (scrollAttempts % 50 === 0) { + console.log( + `Progress: ${followingData.length} users captured after ${scrollAttempts} scroll attempts...` + ); + } + } + + console.log(`Total users captured: ${followingData.length}`); + + return { + usernames: followingUsernames.slice(0, maxUsers), + fullData: followingData.slice(0, maxUsers), + }; +} + +async function scrapeProfile(page, username) { + console.log(`Scraping profile: @${username}`); + + let profileData = { username }; + let dataCapture = false; + + // Set up response listener to intercept API calls + const responseHandler = async (response) => { + const url = response.url(); + + try { + // Check for GraphQL or REST API endpoints + if ( + url.includes("/api/v1/users/web_profile_info/") || + url.includes("/graphql/query") + ) { + const contentType = response.headers()["content-type"] || ""; + if (!contentType.includes("json")) return; + + const json = await response.json(); + + // Handle web_profile_info endpoint (REST API) + if (url.includes("web_profile_info") && json.data?.user) { + if (dataCapture) return; // Already captured, skip duplicate + + const user = json.data.user; + profileData = { + username: user.username, + full_name: user.full_name, + bio: user.biography || "", + followerCount: user.edge_followed_by?.count || 0, + followingCount: user.edge_follow?.count || 0, + profile_pic_url: + user.hd_profile_pic_url_info?.url || user.profile_pic_url, + is_verified: user.is_verified, + is_private: user.is_private, + is_business: user.is_business_account, + category: user.category_name, + external_url: user.external_url, + email: null, + phone: null, + }; + + // Extract email/phone from bio + if (profileData.bio) { + const emailMatch = profileData.bio.match( + /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/ + ); + profileData.email = emailMatch ? emailMatch[0] : null; + + const phoneMatch = profileData.bio.match( + /(\+\d{1,3}[- ]?)?\d{10,14}/ + ); + profileData.phone = phoneMatch ? phoneMatch[0] : null; + } + + dataCapture = true; + } + // Handle GraphQL endpoint + else if (url.includes("graphql") && json.data?.user) { + if (dataCapture) return; // Already captured, skip duplicate + + const user = json.data.user; + profileData = { + username: user.username, + full_name: user.full_name, + bio: user.biography || "", + followerCount: user.follower_count || 0, + followingCount: user.following_count || 0, + profile_pic_url: + user.hd_profile_pic_url_info?.url || user.profile_pic_url, + is_verified: user.is_verified, + is_private: user.is_private, + is_business: user.is_business_account || user.is_business, + category: user.category_name || user.category, + external_url: user.external_url, + email: null, + phone: null, + }; + + // Extract email/phone from bio + if (profileData.bio) { + const emailMatch = profileData.bio.match( + /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/ + ); + profileData.email = emailMatch ? emailMatch[0] : null; + + const phoneMatch = profileData.bio.match( + /(\+\d{1,3}[- ]?)?\d{10,14}/ + ); + profileData.phone = phoneMatch ? phoneMatch[0] : null; + } + + dataCapture = true; + } + } + } catch (e) { + // Ignore errors from parsing non-JSON responses + } + }; + + page.on("response", responseHandler); + + // Navigate to profile page + await handleRateLimitedRequest( + page, + async () => { + await page.goto(`${INSTAGRAM_URL}/${username}/`, { + waitUntil: "domcontentloaded", + }); + }, + `while loading profile @${username}` + ); + + // Wait for API calls to complete + await randomSleep(2000, 3000); + + // Remove listener + page.off("response", responseHandler); + + // If API capture worked, return the data + if (dataCapture) { + return profileData; + } + + // Otherwise, fall back to DOM scraping + console.log(`⚠️ API capture failed for @${username}, using DOM fallback...`); + return await scrapeProfileFallback(page, username); +} + +// Fallback function using DOM scraping +async function scrapeProfileFallback(page, username) { + console.log(`Using DOM scraping for @${username}...`); + + const domData = await page.evaluate(() => { + // Try multiple selectors for bio + let bio = ""; + const bioSelectors = [ + "span._ap3a._aaco._aacu._aacx._aad7._aade", // Updated bio class (2025) + "span._ap3a._aaco._aacu._aacx._aad6._aade", // Previous bio class + "div._aacl._aaco._aacu._aacx._aad7._aade", // Alternative bio with _aad7 + "div._aacl._aaco._aacu._aacx._aad6._aade", // Alternative bio with _aad6 + "h1 + div span", // Bio after username + "header section div span", // Generic header bio + 'div.x7a106z span[dir="auto"]', // Bio container with dir attribute + ]; + + for (const selector of bioSelectors) { + const elem = document.querySelector(selector); + if (elem && elem.innerText && elem.innerText.length > 3) { + bio = elem.innerText; + break; + } + } + + // Get follower/following counts using href-based selectors (stable) + let followerCount = 0; + let followingCount = 0; + + // Method 1: Find by href (most reliable) + const followersLink = document.querySelector('a[href*="/followers/"]'); + const followingLink = document.querySelector('a[href*="/following/"]'); + + if (followersLink) { + const text = followersLink.innerText || followersLink.textContent || ""; + const match = text.match(/[\d,\.]+/); + if (match) { + followerCount = match[0].replace(/,/g, "").replace(/\./g, ""); + } + } + + if (followingLink) { + const text = followingLink.innerText || followingLink.textContent || ""; + const match = text.match(/[\d,\.]+/); + if (match) { + followingCount = match[0].replace(/,/g, "").replace(/\./g, ""); + } + } + + // Alternative: Look in meta tags if href method fails + if (!followerCount) { + const metaContent = + document.querySelector('meta[property="og:description"]')?.content || + ""; + const followerMatch = metaContent.match(/([\d,\.KMB]+)\s+Followers/i); + const followingMatch = metaContent.match(/([\d,\.KMB]+)\s+Following/i); + + if (followerMatch) followerCount = followerMatch[1].replace(/,/g, ""); + if (followingMatch) followingCount = followingMatch[1].replace(/,/g, ""); + } + + // Extract email/phone from bio + let emailMatch = bio.match( + /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/ + ); + let email = emailMatch ? emailMatch[0] : null; + let phoneMatch = bio.match(/(\+\d{1,3}[- ]?)?\d{10,14}/); + let phone = phoneMatch ? phoneMatch[0] : null; + + return { + bio, + followerCount: parseInt(followerCount) || 0, + followingCount: parseInt(followingCount) || 0, + email, + phone, + }; + }); + + return { + username, + ...domData, + }; +} + +async function cronJobs(fn, intervalSec, stopAfter = 0) { + let runCount = 0; + let stop = false; + const timer = setInterval(async () => { + if (stop || (stopAfter && runCount >= stopAfter)) { + clearInterval(timer); + return; + } + await fn(); + runCount++; + }, intervalSec * 1000); + return () => { + stop = true; + }; +} + +async function scrapeWorkflow( + creds, + targetUsername, + proxy = null, + maxFollowingToScrape = 10 +) { + const { browser, page } = await login(creds, proxy); + try { + // Extract current session details for persistence + const session = await extractSession(page); + + // Grab followings with full data + const followingsData = await getFollowingsList( + page, + targetUsername, + maxFollowingToScrape + ); + + console.log( + `Processing ${followingsData.usernames.length} following accounts...` + ); + + for (let i = 0; i < followingsData.usernames.length; i++) { + // Add occasional longer breaks to simulate human behavior + if (i > 0 && i % 10 === 0) { + console.log(`Taking a human-like break after ${i} profiles...`); + await simulateHumanBehavior(page, { mouseMovements: 5, scrolls: 3 }); + await randomSleep(5000, 10000); // Longer break every 10 profiles + } + + const profileInfo = await scrapeProfile( + page, + followingsData.usernames[i] + ); + console.log(JSON.stringify(profileInfo)); + // Implement rate limiting + anti-bot sleep + await randomSleep(2500, 6000); + } + + // Optionally return the full data for further processing + return { + session, + followingsFullData: followingsData.fullData, + scrapedProfiles: followingsData.usernames.length, + }; + } catch (err) { + console.error("Scrape error:", err); + } finally { + await browser.close(); + } +} + +module.exports = { + loginWithSession, + extractSession, + scrapeWorkflow, + getFollowingsList, + scrapeProfile, + cronJobs, +}; diff --git a/server.js b/server.js new file mode 100644 index 0000000..ffc4ad1 --- /dev/null +++ b/server.js @@ -0,0 +1,356 @@ +const { + loginWithSession, + extractSession, + scrapeWorkflow, + getFollowingsList, + scrapeProfile, + cronJobs, +} = require("./scraper.js"); +const { randomSleep, simulateHumanBehavior } = require("./utils.js"); +const fs = require("fs"); +require("dotenv").config(); + +// Full workflow: Login, browse, scrape followings and profiles +async function fullScrapingWorkflow() { + console.log("Starting Instagram Full Scraping Workflow...\n"); + + // Start total timer + const totalStartTime = Date.now(); + + const credentials = { + username: process.env.INSTAGRAM_USERNAME || "your_username", + password: process.env.INSTAGRAM_PASSWORD || "your_password", + }; + + const targetUsername = process.env.TARGET_USERNAME || "instagram"; + const maxFollowing = parseInt(process.env.MAX_FOLLOWING || "20", 10); + const maxProfilesToScrape = parseInt(process.env.MAX_PROFILES || "5", 10); + const proxy = process.env.PROXY || null; + + let browser, page; + + try { + console.log("Configuration:"); + console.log(` Target: @${targetUsername}`); + console.log(` Max following to fetch: ${maxFollowing}`); + console.log(` Max profiles to scrape: ${maxProfilesToScrape}`); + console.log(` Proxy: ${proxy || "None"}\n`); + + // Step 1: Login (with session reuse) + console.log("Step 1: Logging in to Instagram..."); + const loginResult = await loginWithSession(credentials, proxy, true); + browser = loginResult.browser; + page = loginResult.page; + + if (loginResult.sessionReused) { + console.log("Reused existing session!\n"); + } else { + console.log("Fresh login successful!\n"); + } + + // Step 2: Extract and save session + console.log("Step 2: Extracting session cookies..."); + const session = await extractSession(page); + fs.writeFileSync("session_cookies.json", JSON.stringify(session, null, 2)); + console.log(`Session saved (${session.cookies.length} cookies)\n`); + + // Step 3: Simulate browsing before scraping + console.log("Step 3: Simulating human browsing behavior..."); + await simulateHumanBehavior(page, { mouseMovements: 5, scrolls: 3 }); + await randomSleep(2000, 4000); + console.log("Browsing simulation complete\n"); + + // Step 4: Get followings list + console.log(`👥 Step 4: Fetching following list for @${targetUsername}...`); + const followingsStartTime = Date.now(); + + const followingsData = await getFollowingsList( + page, + targetUsername, + maxFollowing + ); + + const followingsEndTime = Date.now(); + const followingsTime = ( + (followingsEndTime - followingsStartTime) / + 1000 + ).toFixed(2); + + console.log( + `✓ Captured ${followingsData.fullData.length} followings in ${followingsTime}s\n` + ); + + // Save followings data + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const followingsFile = `followings_${targetUsername}_${timestamp}.json`; + fs.writeFileSync( + followingsFile, + JSON.stringify( + { + targetUsername, + scrapedAt: new Date().toISOString(), + totalFollowings: followingsData.fullData.length, + followings: followingsData.fullData, + }, + null, + 2 + ) + ); + console.log(`Followings data saved to: ${followingsFile}\n`); + + // Step 5: Scrape individual profiles + console.log( + `📊 Step 5: Scraping ${maxProfilesToScrape} individual profiles...` + ); + const profilesStartTime = Date.now(); + const profilesData = []; + const usernamesToScrape = followingsData.usernames.slice( + 0, + maxProfilesToScrape + ); + + for (let i = 0; i < usernamesToScrape.length; i++) { + const username = usernamesToScrape[i]; + console.log( + ` [${i + 1}/${usernamesToScrape.length}] Scraping @${username}...` + ); + + try { + const profileData = await scrapeProfile(page, username); + profilesData.push(profileData); + console.log(` @${username}: ${profileData.followerCount} followers`); + + // Human-like delay between profiles + await randomSleep(3000, 6000); + + // Take a longer break every 3 profiles + if ((i + 1) % 3 === 0 && i < usernamesToScrape.length - 1) { + console.log(" ⏸ Taking a human-like break..."); + await simulateHumanBehavior(page, { mouseMovements: 4, scrolls: 2 }); + await randomSleep(8000, 12000); + } + } catch (error) { + console.log(` Failed to scrape @${username}: ${error.message}`); + } + } + + const profilesEndTime = Date.now(); + const profilesTime = ((profilesEndTime - profilesStartTime) / 1000).toFixed( + 2 + ); + + console.log( + `\n✓ Scraped ${profilesData.length} profiles in ${profilesTime}s\n` + ); + + // Step 6: Save profiles data + console.log("Step 6: Saving profile data..."); + const profilesFile = `profiles_${targetUsername}_${timestamp}.json`; + fs.writeFileSync( + profilesFile, + JSON.stringify( + { + targetUsername, + scrapedAt: new Date().toISOString(), + totalProfiles: profilesData.length, + profiles: profilesData, + }, + null, + 2 + ) + ); + console.log(`Profiles data saved to: ${profilesFile}\n`); + + // Calculate total time + const totalEndTime = Date.now(); + const totalTime = ((totalEndTime - totalStartTime) / 1000).toFixed(2); + const totalMinutes = Math.floor(totalTime / 60); + const totalSeconds = (totalTime % 60).toFixed(2); + + // Step 7: Summary + console.log("=".repeat(60)); + console.log("📊 SCRAPING SUMMARY"); + console.log("=".repeat(60)); + console.log(`✓ Logged in successfully`); + console.log(`✓ Session cookies saved`); + console.log( + `✓ ${followingsData.fullData.length} followings captured in ${followingsTime}s` + ); + console.log( + `✓ ${profilesData.length} profiles scraped in ${profilesTime}s` + ); + console.log(`\n📁 Files created:`); + console.log(` • ${followingsFile}`); + console.log(` • ${profilesFile}`); + console.log(` • session_cookies.json`); + console.log( + `\n⏱️ Total execution time: ${totalMinutes}m ${totalSeconds}s` + ); + console.log("=".repeat(60) + "\n"); + + return { + success: true, + followingsCount: followingsData.fullData.length, + profilesCount: profilesData.length, + followingsData: followingsData.fullData, + profilesData, + session, + timings: { + followingsTime: parseFloat(followingsTime), + profilesTime: parseFloat(profilesTime), + totalTime: parseFloat(totalTime), + }, + }; + } catch (error) { + console.error("\nScraping workflow failed:"); + console.error(error.message); + console.error(error.stack); + throw error; + } finally { + if (browser) { + console.log("Closing browser..."); + await browser.close(); + console.log("Browser closed\n"); + } + } +} + +// Alternative: Use the built-in scrapeWorkflow function +async function simpleWorkflow() { + console.log("Starting Simple Scraping Workflow (using scrapeWorkflow)...\n"); + + const credentials = { + username: process.env.INSTAGRAM_USERNAME || "your_username", + password: process.env.INSTAGRAM_PASSWORD || "your_password", + }; + + const targetUsername = process.env.TARGET_USERNAME || "instagram"; + const maxFollowing = parseInt(process.env.MAX_FOLLOWING || "20", 10); + const proxy = process.env.PROXY || null; + + try { + console.log(`Target: @${targetUsername}`); + console.log(`Max following to scrape: ${maxFollowing}`); + console.log(`Using proxy: ${proxy || "None"}\n`); + + const result = await scrapeWorkflow( + credentials, + targetUsername, + proxy, + maxFollowing + ); + + console.log("\nScraping completed successfully!"); + console.log(`Total profiles scraped: ${result.scrapedProfiles}`); + console.log( + `Full following data captured: ${result.followingsFullData.length} users` + ); + + // Save the data + if (result.followingsFullData.length > 0) { + const timestamp = new Date().toISOString().replace(/[:.]/g, "-"); + const filename = `scraped_data_${targetUsername}_${timestamp}.json`; + + fs.writeFileSync( + filename, + JSON.stringify( + { + targetUsername, + scrapedAt: new Date().toISOString(), + totalUsers: result.followingsFullData.length, + data: result.followingsFullData, + }, + null, + 2 + ) + ); + + console.log(`Data saved to: ${filename}`); + } + + return result; + } catch (error) { + console.error("\nScraping failed:"); + console.error(error.message); + throw error; + } +} + +// Scheduled scraping with cron +async function scheduledScraping() { + console.log("Starting Scheduled Scraping...\n"); + + const credentials = { + username: process.env.INSTAGRAM_USERNAME || "your_username", + password: process.env.INSTAGRAM_PASSWORD || "your_password", + }; + + const targetUsername = process.env.TARGET_USERNAME || "instagram"; + const intervalMinutes = parseInt(process.env.SCRAPE_INTERVAL || "60", 10); + const maxRuns = parseInt(process.env.MAX_RUNS || "5", 10); + + console.log( + `Will scrape @${targetUsername} every ${intervalMinutes} minutes` + ); + console.log(`Maximum runs: ${maxRuns}\n`); + + let runCount = 0; + + const stopCron = await cronJobs( + async () => { + runCount++; + console.log(`\n${"=".repeat(60)}`); + console.log( + `📅 Scheduled Run #${runCount} - ${new Date().toLocaleString()}` + ); + console.log("=".repeat(60)); + + try { + await simpleWorkflow(); + } catch (error) { + console.error(`Run #${runCount} failed:`, error.message); + } + + if (runCount >= maxRuns) { + console.log(`\nCompleted ${maxRuns} scheduled runs. Stopping...`); + process.exit(0); + } + }, + intervalMinutes * 60, // Convert to seconds + maxRuns + ); + + console.log("Cron job started. Press Ctrl+C to stop.\n"); +} + +// Main entry point +if (require.main === module) { + const mode = process.env.MODE || "full"; // full, simple, or scheduled + + console.log(`Mode: ${mode}\n`); + + let workflow; + if (mode === "simple") { + workflow = simpleWorkflow(); + } else if (mode === "scheduled") { + workflow = scheduledScraping(); + } else { + workflow = fullScrapingWorkflow(); + } + + workflow + .then(() => { + console.log("All done!"); + process.exit(0); + }) + .catch((err) => { + console.error("\nFatal error:", err); + process.exit(1); + }); +} + +module.exports = { + fullScrapingWorkflow, + simpleWorkflow, + scheduledScraping, +}; diff --git a/utils.js b/utils.js new file mode 100644 index 0000000..ca0ee24 --- /dev/null +++ b/utils.js @@ -0,0 +1,146 @@ +function randomSleep(minMs = 2000, maxMs = 5000) { + const delay = Math.floor(Math.random() * (maxMs - minMs + 1)) + minMs; + return new Promise((res) => setTimeout(res, delay)); +} + +async function humanLikeMouseMovement(page, steps = 10) { + // Simulate human-like mouse movements across the page + const viewport = await page.viewport(); + const width = viewport.width; + const height = viewport.height; + + for (let i = 0; i < steps; i++) { + const x = Math.floor(Math.random() * width); + const y = Math.floor(Math.random() * height); + + await page.mouse.move(x, y, { steps: Math.floor(Math.random() * 10) + 5 }); + await randomSleep(100, 500); + } +} + +async function randomScroll(page, scrollCount = 3) { + // Perform random scrolling to simulate human behavior + for (let i = 0; i < scrollCount; i++) { + const scrollAmount = Math.floor(Math.random() * 300) + 100; + await page.evaluate((amount) => { + window.scrollBy(0, amount); + }, scrollAmount); + await randomSleep(800, 1500); + } +} + +async function simulateHumanBehavior(page, options = {}) { + // Combined function to simulate various human-like behaviors + const { mouseMovements = 5, scrolls = 2, randomClicks = false } = options; + + // Random mouse movements + if (mouseMovements > 0) { + await humanLikeMouseMovement(page, mouseMovements); + } + + // Random scrolling + if (scrolls > 0) { + await randomScroll(page, scrolls); + } + + // Optional: Random clicks on non-interactive elements + if (randomClicks) { + try { + await page.evaluate(() => { + const elements = document.querySelectorAll("div, span, p"); + if (elements.length > 0) { + const randomElement = + elements[Math.floor(Math.random() * elements.length)]; + const rect = randomElement.getBoundingClientRect(); + // Just move to it, don't actually click to avoid triggering actions + } + }); + } catch (err) { + // Ignore errors from random element selection + } + } + + await randomSleep(500, 1000); +} + +async function withRetry(fn, options = {}) { + const { + maxRetries = 3, + initialDelay = 2000, + maxDelay = 30000, + shouldRetry = (error) => true, + } = options; + + for (let attempt = 0; attempt < maxRetries; attempt++) { + try { + return await fn(); + } catch (error) { + const isLastAttempt = attempt === maxRetries - 1; + + // Check if we should retry this error + if (!shouldRetry(error) || isLastAttempt) { + throw error; + } + + // Calculate exponential backoff delay: 2s, 4s, 8s, 16s, 30s (capped) + const exponentialDelay = Math.min( + initialDelay * Math.pow(2, attempt), + maxDelay + ); + + // Add jitter (randomize ±20%) to avoid thundering herd + const jitter = exponentialDelay * (0.8 + Math.random() * 0.4); + const delay = Math.floor(jitter); + + console.log( + `Retry attempt ${attempt + 1}/${maxRetries} after ${delay}ms delay...` + ); + console.log(`Error: ${error.message || error}`); + + await randomSleep(delay, delay); + } + } +} + +async function handleRateLimitedRequest(page, requestFn, context = "") { + return withRetry(requestFn, { + maxRetries: 5, + initialDelay: 2000, + maxDelay: 60000, + shouldRetry: (error) => { + // Retry on rate limit (429) or temporary errors + if (error.status === 429 || error.statusCode === 429) { + console.log(`Rate limited (429) ${context}. Backing off...`); + return true; + } + + // Retry on 5xx server errors + if (error.status >= 500 || error.statusCode >= 500) { + console.log( + `Server error (${ + error.status || error.statusCode + }) ${context}. Retrying...` + ); + return true; + } + + // Retry on network errors + if (error.code === "ECONNRESET" || error.code === "ETIMEDOUT") { + console.log(`Network error (${error.code}) ${context}. Retrying...`); + return true; + } + + // Don't retry on client errors (4xx except 429) + return false; + }, + }); +} + +module.exports = { + randomSleep, + humanLikeMouseMovement, + randomScroll, + simulateHumanBehavior, + withRetry, + handleRateLimitedRequest, +};