Create a Python web scraper using the BeautifulSoup library that extracts data from a specified website (e.g., product prices, news headlines) and saves it to a JSON file. The script should handle pagination and include error handling for cases when the target website is unavailable or its structure changes.
PLAINFORMAT
import requests
from bs4 import BeautifulSoup
import json

# Define the target URL and headers
url = 'https://www.example.com/news'
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}

def get_webpage_content(url):
    try:
        response = requests.get(url, headers=headers)
        response.raise_for_status()
        return response.text
    except requests.exceptions.RequestException as e:
        print(f"Error requesting {url}: {str(e)}")
        return None

def parse_webpage(html):
    soup = BeautifulSoup(html, 'html.parser')
    return soup

def extract_headlines(soup):
    headlines = []
    try:
        # Replace the CSS selector below with the appropriate one for your target website
        headline_elements = soup.select('.headline-class')
        for element in headline_elements:
            headlines.append(element.get_text())
    except Exception as e:
        print(f"Error extracting headlines: {str(e)}")
    return headlines

def save_to_json(data, filename):
    with open(filename, 'w') as f:
        json.dump(data, f)

if __name__ == "__main__":
    html = get_webpage_content(url)
    if html:
        soup = parse_webpage(html)
        headlines = extract_headlines(soup)
        if headlines:
            save_to_json(headlines, 'headlines.json')
        else:
            print("No headlines found.")
    else:
        print("Failed to fetch webpage content.")