import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import csv
import time
import json
from datetime import datetime, timedelta

class EnhancedSEOCrawler:
    def __init__(self, base_url, gsc_credentials=None):
        self.base_url = base_url
        self.visited_urls = set()
        self.seo_issues = []
        self.gsc_credentials = gsc_credentials
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        })
        
    def crawl_page(self, url, max_depth=3, current_depth=0):
        """Enhanced crawling with depth control and rate limiting"""
        if url in self.visited_urls or current_depth > max_depth:
            return
        
        try:
            print(f"Crawling: {url}")
            response = self.session.get(url, timeout=15)
            self.visited_urls.add(url)
            
            if response.status_code == 200:
                soup = BeautifulSoup(response.content, 'html.parser')
                self.analyze_page_seo(soup, url, response)
                
                # Follow internal links for deeper crawling
                if current_depth < max_depth:
                    internal_links = self.get_internal_links(soup, url)
                    for link in internal_links[:10]:  # Limit to 10 links per page
                        time.sleep(1)  # Rate limiting
                        self.crawl_page(link, max_depth, current_depth + 1)
                        
        except Exception as e:
            self.seo_issues.append({
                'url': url,
                'issue': f'Error crawling: {str(e)}',
                'type': 'crawl_error',
                'timestamp': datetime.now().isoformat()
            })
    
    def analyze_page_seo(self, soup, url, response):
        """Comprehensive SEO analysis"""
        issues = []
        
        # Title analysis
        title = soup.find('title')
        if not title:
            issues.append('Missing title tag')
        elif len(title.text.strip()) < 10:
            issues.append('Title too short (< 10 characters)')
        elif len(title.text.strip()) > 60:
            issues.append('Title too long (> 60 characters)')
        
        # Meta description analysis
        meta_desc = soup.find('meta', attrs={'name': 'description'})
        if not meta_desc:
            issues.append('Missing meta description')
        elif len(meta_desc.get('content', '')) < 50:
            issues.append('Meta description too short (< 50 characters)')
        elif len(meta_desc.get('content', '')) > 160:
            issues.append('Meta description too long (> 160 characters)')
        
        # Heading structure analysis
        headings = soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])
        h1_count = len(soup.find_all('h1'))
        if h1_count == 0:
            issues.append('Missing H1 tag')
        elif h1_count > 1:
            issues.append('Multiple H1 tags found')
        
        # Image analysis
        images = soup.find_all('img')
        images_without_alt = [img for img in images if not img.get('alt')]
        if images_without_alt:
            issues.append(f'{len(images_without_alt)} images without alt text')
        
        # Internal link analysis
        internal_links = self.get_internal_links(soup, url)
        if len(internal_links) < 3:
            issues.append('Few internal links (< 3)')
        
        # Schema markup check
        schema_scripts = soup.find_all('script', type='application/ld+json')
        if not schema_scripts:
            issues.append('No structured data found')
        
        # Add all issues to the main list
        for issue in issues:
            self.seo_issues.append({
                'url': url,
                'issue': issue,
                'type': 'seo_issue',
                'timestamp': datetime.now().isoformat(),
                'status_code': response.status_code,
                'content_length': len(response.content)
            })
    
    def get_internal_links(self, soup, current_url):
        """Extract internal links from page"""
        internal_links = []
        links = soup.find_all('a', href=True)
        
        for link in links:
            href = link['href']
            if href.startswith('/') or self.base_url in href:
                full_url = urljoin(current_url, href)
                if full_url not in self.visited_urls:
                    internal_links.append(full_url)
        
        return list(set(internal_links))
    
    def check_link_status(self, url):
        """Check if URL is accessible"""
        try:
            response = self.session.head(url, timeout=10)
            return response.status_code < 400
        except:
            return False
    
    def export_results(self, filename='enhanced_seo_audit.csv'):
        """Export results with enhanced data"""
        with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
            fieldnames = ['url', 'issue', 'type', 'timestamp', 'status_code', 'content_length']
            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
            writer.writeheader()
            writer.writerows(self.seo_issues)
        
        print(f"Audit completed. Found {len(self.seo_issues)} issues across {len(self.visited_urls)} pages.")
        print(f"Results exported to: {filename}")

# Usage
crawler = EnhancedSEOCrawler('https://example.com')
crawler.crawl_page('https://example.com', max_depth=2)
crawler.export_results()

import subprocess
import json
import pandas as pd
from datetime import datetime

class LighthouseAnalyzer:
    def __init__(self):
        self.results = []
    
    def run_lighthouse(self, url, output_format='json'):
        """Run Lighthouse audit on a URL"""
        try:
            cmd = [
                'lighthouse',
                url,
                f'--output={output_format}',
                '--output-path=./lighthouse-report.json',
                '--chrome-flags="--headless"',
                '--only-categories=performance'
            ]
            
            subprocess.run(cmd, check=True)
            
            with open('./lighthouse-report.json', 'r') as f:
                data = json.load(f)
            
            return self.extract_web_vitals(data)
            
        except Exception as e:
            print(f"Error running Lighthouse: {e}")
            return None
    
    def extract_web_vitals(self, lighthouse_data):
        """Extract Core Web Vitals from Lighthouse results"""
        audits = lighthouse_data.get('audits', {})
        
        return {
            'url': lighthouse_data.get('finalUrl', ''),
            'timestamp': datetime.now().isoformat(),
            'lcp': audits.get('largest-contentful-paint', {}).get('numericValue', 0),
            'fid': audits.get('max-potential-fid', {}).get('numericValue', 0),
            'cls': audits.get('cumulative-layout-shift', {}).get('numericValue', 0),
            'ttfb': audits.get('server-response-time', {}).get('numericValue', 0),
            'performance_score': lighthouse_data.get('categories', {}).get('performance', {}).get('score', 0) * 100
        }
    
    def analyze_multiple_urls(self, urls):
        """Analyze multiple URLs and compile results"""
        for url in urls:
            result = self.run_lighthouse(url)
            if result:
                self.results.append(result)
        
        return pd.DataFrame(self.results)
    
    def generate_report(self, df, output_file='web_vitals_report.csv'):
        """Generate CSV report with recommendations"""
        df['lcp_status'] = df['lcp'].apply(lambda x: 'Good' if x < 2500 else 'Needs Improvement' if x < 4000 else 'Poor')
        df['fid_status'] = df['fid'].apply(lambda x: 'Good' if x < 100 else 'Needs Improvement' if x < 300 else 'Poor')
        df['cls_status'] = df['cls'].apply(lambda x: 'Good' if x < 0.1 else 'Needs Improvement' if x < 0.25 else 'Poor')
        
        df.to_csv(output_file, index=False)
        return df

# Usage
analyzer = LighthouseAnalyzer()
urls = ['https://example.com', 'https://example.com/page1', 'https://example.com/page2']
results = analyzer.analyze_multiple_urls(urls)
report = analyzer.generate_report(results)

import requests
from bs4 import BeautifulSoup
import json
import re

class SchemaValidator:
    def __init__(self):
        self.schema_types = [
            'Organization', 'Product', 'Article', 'LocalBusiness', 
            'FAQ', 'BreadcrumbList', 'WebSite'
        ]
    
    def extract_schema(self, url):
        """Extract and validate schema markup from a URL"""
        try:
            response = requests.get(url)
            soup = BeautifulSoup(response.content, 'html.parser')
            
            # Find JSON-LD schema
            json_ld_scripts = soup.find_all('script', type='application/ld+json')
            microdata = soup.find_all(attrs={'itemtype': True})
            
            results = {
                'url': url,
                'json_ld_schemas': [],
                'microdata_schemas': [],
                'validation_errors': []
            }
            
            # Process JSON-LD
            for script in json_ld_scripts:
                try:
                    schema_data = json.loads(script.string)
                    schema_type = self.get_schema_type(schema_data)
                    results['json_ld_schemas'].append({
                        'type': schema_type,
                        'data': schema_data,
                        'valid': self.validate_schema(schema_data, schema_type)
                    })
                except json.JSONDecodeError:
                    results['validation_errors'].append('Invalid JSON-LD format')
            
            # Process Microdata
            for item in microdata:
                itemtype = item.get('itemtype', '')
                schema_type = itemtype.split('/')[-1] if '/' in itemtype else itemtype
                results['microdata_schemas'].append({
                    'type': schema_type,
                    'element': str(item)[:200] + '...' if len(str(item)) > 200 else str(item)
                })
            
            return results
            
        except Exception as e:
            return {'url': url, 'error': str(e)}
    
    def get_schema_type(self, schema_data):
        """Extract schema type from JSON-LD data"""
        if isinstance(schema_data, dict):
            if '@type' in schema_data:
                return schema_data['@type']
            elif 'type' in schema_data:
                return schema_data['type']
        elif isinstance(schema_data, list):
            for item in schema_data:
                if isinstance(item, dict) and '@type' in item:
                    return item['@type']
        return 'Unknown'
    
    def validate_schema(self, schema_data, schema_type):
        """Basic schema validation"""
        required_fields = {
            'Organization': ['name'],
            'Product': ['name', 'description'],
            'Article': ['headline', 'author'],
            'LocalBusiness': ['name', 'address']
        }
        
        if schema_type in required_fields:
            for field in required_fields[schema_type]:
                if not self.has_field(schema_data, field):
                    return False
        return True
    
    def has_field(self, data, field):
        """Check if field exists in schema data"""
        if isinstance(data, dict):
            return field in data
        elif isinstance(data, list):
            return any(self.has_field(item, field) for item in data)
        return False
    
    def generate_schema_report(self, urls):
        """Generate comprehensive schema markup report"""
        report = []
        for url in urls:
            result = self.extract_schema(url)
            report.append(result)
        return report

# Usage
validator = SchemaValidator()
urls = ['https://example.com', 'https://example.com/products']
schema_report = validator.generate_schema_report(urls)

url,lastmod,changefreq,priority,accessible
https://example.com/page1,2024-01-01,daily,0.8,TRUE
https://example.com/page2,2024-01-02,weekly,0.5,FALSE

import requests
import xml.etree.ElementTree as ET
from urllib.parse import urljoin, urlparse
import pandas as pd

class SitemapAnalyzer:
    def __init__(self, base_url):
        self.base_url = base_url
        self.sitemap_urls = []
        self.page_data = []
    
    def discover_sitemaps(self):
        """Find sitemaps for the website"""
        # Check robots.txt for sitemap location
        robots_url = urljoin(self.base_url, '/robots.txt')
        try:
            response = requests.get(robots_url)
            if response.status_code == 200:
                for line in response.text.split('
'):
                    if line.lower().startswith('sitemap:'):
                        sitemap_url = line.split(':', 1)[1].strip()
                        self.sitemap_urls.append(sitemap_url)
        except:
            pass
        
        # Try common sitemap locations
        common_sitemaps = [
            '/sitemap.xml',
            '/sitemap_index.xml',
            '/sitemap1.xml'
        ]
        
        for sitemap in common_sitemaps:
            sitemap_url = urljoin(self.base_url, sitemap)
            try:
                response = requests.head(sitemap_url)
                if response.status_code == 200:
                    self.sitemap_urls.append(sitemap_url)
            except:
                continue
    
    def analyze_sitemap(self, sitemap_url):
        """Analyze a single sitemap"""
        try:
            response = requests.get(sitemap_url)
            root = ET.fromstring(response.content)
            
            # Handle sitemap index
            if 'sitemapindex' in root.tag:
                for sitemap in root.findall('.//{*}sitemap/{*}loc'):
                    self.analyze_sitemap(sitemap.text)
            else:
                # Handle URL sitemap
                for url in root.findall('.//{*}url'):
                    url_data = {}
                    
                    loc = url.find('{*}loc')
                    if loc is not None:
                        url_data['url'] = loc.text
                    
                    lastmod = url.find('{*}lastmod')
                    if lastmod is not None:
                        url_data['lastmod'] = lastmod.text
                    
                    changefreq = url.find('{*}changefreq')
                    if changefreq is not None:
                        url_data['changefreq'] = changefreq.text
                    
                    priority = url.find('{*}priority')
                    if priority is not None:
                        url_data['priority'] = float(priority.text)
                    
                    # Check if URL is accessible
                    url_data['accessible'] = self.check_url_accessibility(url_data['url'])
                    
                    self.page_data.append(url_data)
                    
        except Exception as e:
            print(f"Error analyzing sitemap {sitemap_url}: {e}")
    
    def check_url_accessibility(self, url):
        """Check if URL returns 200 status code"""
        try:
            response = requests.head(url, timeout=10)
            return response.status_code == 200
        except:
            return False
    
    def generate_sitemap_report(self):
        """Generate comprehensive sitemap analysis report"""
        self.discover_sitemaps()
        
        for sitemap_url in self.sitemap_urls:
            self.analyze_sitemap(sitemap_url)
        
        df = pd.DataFrame(self.page_data)
        
        report = {
            'total_urls': len(df),
            'accessible_urls': df['accessible'].sum() if 'accessible' in df.columns else 0,
            'broken_urls': len(df) - (df['accessible'].sum() if 'accessible' in df.columns else 0),
            'avg_priority': df['priority'].mean() if 'priority' in df.columns else 0,
            'changefreq_distribution': df['changefreq'].value_counts().to_dict() if 'changefreq' in df.columns else {},
            'urls_without_lastmod': df['lastmod'].isna().sum() if 'lastmod' in df.columns else 0
        }
        
        return report, df

# Usage
analyzer = SitemapAnalyzer('https://example.com')
report, data = analyzer.generate_sitemap_report()
print(f"Total URLs: {report['total_urls']}")
print(f"Accessible URLs: {report['accessible_urls']}")
print(f"Broken URLs: {report['broken_urls']}")

from google.oauth2 import service_account
from googleapiclient.discovery import build
import pandas as pd
from datetime import datetime, timedelta

class GoogleSearchConsoleAPI:
    def __init__(self, credentials_path):
        self.credentials = service_account.Credentials.from_service_account_file(
            credentials_path,
            scopes=['https://www.googleapis.com/auth/webmasters.readonly']
        )
        self.service = build('searchconsole', 'v1', credentials=self.credentials)
    
    def get_search_analytics(self, site_url, start_date, end_date, dimensions=['query', 'page']):
        """Extract search analytics data"""
        request = {
            'startDate': start_date.strftime('%Y-%m-%d'),
            'endDate': end_date.strftime('%Y-%m-%d'),
            'dimensions': dimensions,
            'rowLimit': 25000
        }
        
        try:
            response = self.service.searchAnalytics().query(
                siteUrl=site_url,
                body=request
            ).execute()
            
            return self.process_search_data(response)
        except Exception as e:
            print(f"Error fetching search analytics: {e}")
            return pd.DataFrame()
    
    def get_index_status(self, site_url):
        """Get indexing status for pages"""
        try:
            response = self.service.urlInspection().index().inspect(
                body={'inspectionUrl': site_url}
            ).execute()
            
            return {
                'url': site_url,
                'index_status': response.get('inspectionResult', {}).get('indexStatusResult', {}).get('verdict', 'Unknown'),
                'coverage_state': response.get('inspectionResult', {}).get('indexStatusResult', {}).get('coverageState', 'Unknown'),
                'robots_txt_state': response.get('inspectionResult', {}).get('indexStatusResult', {}).get('robotsTxtState', 'Unknown')
            }
        except Exception as e:
            print(f"Error checking index status: {e}")
            return None
    
    def get_sitemaps(self, site_url):
        """Get sitemap information"""
        try:
            response = self.service.sitemaps().list(siteUrl=site_url).execute()
            return response.get('sitemap', [])
        except Exception as e:
            print(f"Error fetching sitemaps: {e}")
            return []
    
    def process_search_data(self, response):
        """Process search analytics response into DataFrame"""
        rows = response.get('rows', [])
        data = []
        
        for row in rows:
            keys = row.get('keys', [])
            data.append({
                'query': keys[0] if len(keys) > 0 else '',
                'page': keys[1] if len(keys) > 1 else '',
                'clicks': row.get('clicks', 0),
                'impressions': row.get('impressions', 0),
                'ctr': row.get('ctr', 0),
                'position': row.get('position', 0)
            })
        
        return pd.DataFrame(data)

# Usage
gsc_api = GoogleSearchConsoleAPI('path/to/credentials.json')
end_date = datetime.now()
start_date = end_date - timedelta(days=30)

search_data = gsc_api.get_search_analytics(
    'https://example.com',
    start_date,
    end_date,
    ['query', 'page', 'country']
)

url,title,content_length,word_count,keywords
https://competitor.com/page1,Page Title,5000,800,"keyword1,keyword2,keyword3"
https://competitor.com/page2,Another Title,3000,500,"keyword4,keyword5"

import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
import requests
from bs4 import BeautifulSoup
import json

class ContentResearchTool:
    def __init__(self, api_key=None):
        self.api_key = api_key
        self.content_data = []
        
    def analyze_competitor_content(self, competitor_urls):
        """Analyze competitor content to identify gaps"""
        for url in competitor_urls:
            try:
                response = requests.get(url)
                soup = BeautifulSoup(response.content, 'html.parser')
                
                # Extract content information
                title = soup.find('title').text if soup.find('title') else ''
                h1 = soup.find('h1').text if soup.find('h1') else ''
                content = ' '.join([p.text for p in soup.find_all('p')])
                
                self.content_data.append({
                    'url': url,
                    'title': title,
                    'h1': h1,
                    'content_length': len(content),
                    'word_count': len(content.split()),
                    'keywords': self.extract_keywords(content)
                })
                
            except Exception as e:
                print(f"Error analyzing {url}: {e}")
        
        return pd.DataFrame(self.content_data)
    
    def extract_keywords(self, text, top_n=10):
        """Extract top keywords from text using TF-IDF"""
        vectorizer = TfidfVectorizer(
            max_features=top_n,
            stop_words='english',
            ngram_range=(1, 2)
        )
        
        try:
            tfidf_matrix = vectorizer.fit_transform([text])
            feature_names = vectorizer.get_feature_names_out()
            scores = tfidf_matrix.toarray()[0]
            
            # Return keywords with scores
            keywords = list(zip(feature_names, scores))
            return sorted(keywords, key=lambda x: x[1], reverse=True)
        except:
            return []
    
    def identify_content_gaps(self, your_content_df, competitor_content_df):
        """Identify content gaps between your site and competitors"""
        your_keywords = set()
        competitor_keywords = set()
        
        # Collect all keywords from your content
        for keywords in your_content_df['keywords']:
            your_keywords.update([kw[0] for kw in keywords])
        
        # Collect all keywords from competitor content
        for keywords in competitor_content_df['keywords']:
            competitor_keywords.update([kw[0] for kw in keywords])
        
        # Find gaps
        content_gaps = competitor_keywords - your_keywords
        
        return list(content_gaps)
    
    def create_topic_clusters(self, content_df, n_clusters=5):
        """Group content into topic clusters"""
        # Prepare text for clustering
        text_data = []
        for _, row in content_df.iterrows():
            text = f"{row['title']} {row['h1']} {' '.join([kw[0] for kw in row['keywords']])}"
            text_data.append(text)
        
        # Vectorize text
        vectorizer = TfidfVectorizer(max_features=100, stop_words='english')
        tfidf_matrix = vectorizer.fit_transform(text_data)
        
        # Perform clustering
        kmeans = KMeans(n_clusters=n_clusters, random_state=42)
        clusters = kmeans.fit_predict(tfidf_matrix)
        
        # Add cluster information to dataframe
        content_df['cluster'] = clusters
        
        return content_df
    
    def generate_content_ideas(self, gaps, cluster_analysis):
        """Generate content ideas based on gaps and clusters"""
        content_ideas = []
        
        for gap in gaps[:20]:  # Top 20 gaps
            content_ideas.append({
                'type': 'gap_fill',
                'topic': gap,
                'priority': 'high',
                'suggested_title': f"Complete Guide to {gap.replace('_', ' ').title()}",
                'target_keywords': [gap],
                'estimated_word_count': 1500
            })
        
        # Add cluster-based ideas
        for cluster_id in cluster_analysis['cluster'].unique():
            cluster_content = cluster_analysis[cluster_analysis['cluster'] == cluster_id]
            if len(cluster_content) > 0:
                main_topic = cluster_content.iloc[0]['title']
                content_ideas.append({
                    'type': 'cluster_expansion',
                    'topic': main_topic,
                    'priority': 'medium',
                    'suggested_title': f"Comprehensive {main_topic} Guide",
                    'target_keywords': [kw[0] for kw in cluster_content.iloc[0]['keywords'][:5]],
                    'estimated_word_count': 2000
                })
        
        return content_ideas

# Usage
researcher = ContentResearchTool()
competitor_urls = ['https://competitor1.com', 'https://competitor2.com']
competitor_analysis = researcher.analyze_competitor_content(competitor_urls)
clustered_content = researcher.create_topic_clusters(competitor_analysis)
content_ideas = researcher.generate_content_ideas([], clustered_content)