Create app.py

app.py

@@ -0,0 +1,129 @@
+import streamlit as st
+import requests
+import pandas as pd
+from bs4 import BeautifulSoup
+
+# Helper Functions
+COMMON_SITEMAP_LOCATIONS = [
+    "/sitemap.xml",
+    "/sitemap_index.xml",
+    "/sitemap-index.xml",
+    "/sitemap.php",
+    "/sitemap.txt",
+    "/sitemap.xml.gz",
+    "/sitemap/",
+    "/sitemap/sitemap.xml",
+    "/sitemapindex.xml",
+    "/sitemap/index.xml",
+    "/sitemap1.xml",
+    "/rss/",
+    "/rss.xml",
+    "/atom.xml",
+]
+
+
+def find_sitemap(domain):
+    """Locate the sitemap URL by checking common locations and robots.txt."""
+    for path in COMMON_SITEMAP_LOCATIONS:
+        sitemap_url = domain.rstrip("/") + path
+        try:
+            response = requests.get(sitemap_url, timeout=5)
+            if response.status_code == 200:
+                return sitemap_url
+        except requests.RequestException:
+            continue
+
+    robots_url = domain.rstrip("/") + "/robots.txt"
+    try:
+        response = requests.get(robots_url, timeout=5)
+        if response.status_code == 200:
+            for line in response.text.splitlines():
+                if line.lower().startswith("sitemap:"):
+                    return line.split(":", 1)[1].strip()
+    except requests.RequestException:
+        pass
+    return None
+
+
+def get_sitemap_links(sitemap_url):
+    """Fetch all links from a sitemap."""
+    response = requests.get(sitemap_url)
+    if response.status_code != 200:
+        raise Exception(f"Failed to fetch sitemap: {response.status_code}")
+
+    soup = BeautifulSoup(response.content, "lxml-xml")
+    links = [loc.text.strip() for loc in soup.find_all("loc")]
+    return links
+
+
+def is_blog_link(link):
+    """Filter links for blog-related URLs."""
+    return "blog" in link.lower()
+
+
+def extract_article_info(url):
+    """Extract the article content from a URL."""
+    try:
+        response = requests.get(url, timeout=10)
+        response.raise_for_status()
+        response.encoding = response.apparent_encoding
+        soup = BeautifulSoup(response.text, "html.parser")
+
+        heading = soup.find("h1").get_text(strip=True) if soup.find("h1") else None
+        all_paragraphs = [p.get_text(strip=True) for p in soup.find_all("p")]
+        article_text = "\n\n".join(all_paragraphs)
+        full_article_text = f"{heading}\n\n{article_text}" if heading else article_text
+        return full_article_text
+    except Exception as e:
+        return f"Error fetching article: {e}"
+
+
+# Streamlit App
+st.title("Blog Article Scraper")
+
+# Input Website URL
+website_url = st.text_input("Enter the website URL (e.g., https://example.com):")
+
+if st.button("Start Scraping"):
+    if not website_url:
+        st.error("Please enter a website URL.")
+    else:
+        st.write("Locating sitemap...")
+        sitemap_url = find_sitemap(website_url)
+
+        if not sitemap_url:
+            st.error("Could not locate a sitemap.")
+        else:
+            st.success(f"Sitemap found: {sitemap_url}")
+            st.write("Fetching links from sitemap...")
+            try:
+                links = get_sitemap_links(sitemap_url)
+                blog_links = [link for link in links if is_blog_link(link)]
+                st.success(f"Found {len(blog_links)} blog links.")
+
+                # Scraping articles
+                st.write("Extracting article content...")
+                data = []
+                progress = st.progress(0)
+                for i, link in enumerate(blog_links):
+                    article_text = extract_article_info(link)
+                    data.append({"URL": link, "Article Text": article_text})
+                    progress.progress((i + 1) / len(blog_links))
+
+                # Save results to DataFrame and Display
+                df = pd.DataFrame(data)
+                st.write("Scraping completed.")
+                st.dataframe(df)
+
+                # Provide download link for Excel file
+                file_name = "blog_articles.xlsx"
+                df.to_excel(file_name, index=False)
+                with open(file_name, "rb") as file:
+                    st.download_button(
+                        label="Download Excel File",
+                        data=file,
+                        file_name="blog_articles.xlsx",
+                        mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+                    )
+            except Exception as e:
+                st.error(f"Error during scraping: {e}")