Update part1_data.py

part1_data.py

@@ -53,173 +53,137 @@ class TobaccoAnalyzer:
         return None
 
     def get_weather_data(self, lat, lon, historical_days=90, forecast_days=90):
-        """Get historical and forecast weather data with pattern variations"""
+        """Get historical and forecast weather data"""
         historical_data = []
         
-        # Get current weather and recent history
-        current_url = f"https://api.openweathermap.org/data/2.5/weather?lat={lat}&lon={lon}&appid={self.api_key}&units=metric"
-        try:
-            response = requests.get(current_url)
-            if response.status_code == 200:
-                current = response.json()
-                base_temp = current['main']['temp']
-                base_humidity = current['main']['humidity']
-                base_rainfall = current.get('rain', {}).get('1h', 0) * 24
-            else:
-                base_temp = 25
-                base_humidity = 70
-                base_rainfall = 0
-        except Exception as e:
-            print(f"Error fetching current weather: {e}")
-            base_temp = 25
-            base_humidity = 70
-            base_rainfall = 0
-
-        # Generate historical data with patterns
+        # Get historical data
         for day in range(historical_days):
             date = datetime.now() - timedelta(days=day)
-            
-            # Add daily patterns
-            for hour in range(24):
-                # Temperature pattern: Daily cycle with random variations
-                hour_temp = base_temp + \
-                           3 * np.sin((hour - 6) * np.pi / 12) + \
-                           np.random.normal(0, 1)
-                
-                # Humidity pattern: Inverse to temperature
-                hour_humidity = base_humidity - \
-                              10 * np.sin((hour - 6) * np.pi / 12) + \
-                              np.random.normal(0, 5)
-                
-                # Rainfall pattern: More likely in afternoon
-                rain_chance = 0.1 + 0.2 * np.sin((hour - 12) * np.pi / 12)
-                hour_rainfall = np.random.exponential(base_rainfall) if np.random.random() < rain_chance else 0
-                
-                weather_data = {
-                    'date': date + timedelta(hours=hour),
-                    'temperature': hour_temp,
-                    'humidity': np.clip(hour_humidity, 0, 100),
-                    'rainfall': hour_rainfall,
-                    'type': 'historical',
-                    'description': self.get_weather_description(hour_temp, hour_humidity, hour_rainfall),
-                    'temp_min': hour_temp - np.random.uniform(0, 2),
-                    'temp_max': hour_temp + np.random.uniform(0, 2),
-                    'wind_speed': np.random.normal(5, 2),
-                    'clouds': np.random.normal(50, 20)
-                }
-                historical_data.append(weather_data)
+            url = f"https://api.openweathermap.org/data/2.5/weather?lat={lat}&lon={lon}&appid={self.api_key}&units=metric&dt={int(date.timestamp())}"
+            try:
+                response = requests.get(url)
+                if response.status_code == 200:
+                    data = response.json()
+                    weather_data = {
+                        'date': date,
+                        'temperature': float(data['main']['temp']),  # Ensure numeric
+                        'humidity': float(data['main']['humidity']),
+                        'rainfall': float(data.get('rain', {}).get('1h', 0)) * 24,
+                        'type': 'historical',
+                        'description': data['weather'][0]['description'],
+                        'temp_min': float(data['main']['temp_min']),
+                        'temp_max': float(data['main']['temp_max'])
+                    }
+                    historical_data.append(weather_data)
+            except Exception as e:
+                print(f"Error fetching historical data: {e}")
 
-        # Get 5-day forecast
+        # Get forecast data
         forecast_data = []
-        forecast_url = f"https://api.openweathermap.org/data/2.5/forecast?lat={lat}&lon={lon}&appid={self.api_key}&units=metric"
         try:
+            forecast_url = f"https://api.openweathermap.org/data/2.5/forecast?lat={lat}&lon={lon}&appid={self.api_key}&units=metric"
             response = requests.get(forecast_url)
             if response.status_code == 200:
                 data = response.json()
+                # Group forecast data by day
+                daily_forecasts = {}
+                
                 for item in data['list']:
+                    date = datetime.fromtimestamp(item['dt'])
+                    day_key = date.date()
+                    
+                    if day_key not in daily_forecasts:
+                        daily_forecasts[day_key] = {
+                            'temps': [],
+                            'humidity': [],
+                            'rainfall': 0,
+                            'descriptions': []
+                        }
+                    
+                    daily_forecasts[day_key]['temps'].append(float(item['main']['temp']))
+                    daily_forecasts[day_key]['humidity'].append(float(item['main']['humidity']))
+                    daily_forecasts[day_key]['rainfall'] += float(item.get('rain', {}).get('3h', 0))
+                    daily_forecasts[day_key]['descriptions'].append(item['weather'][0]['description'])
+
+                # Create daily forecast entries
+                for day_key, day_data in daily_forecasts.items():
                     forecast = {
-                        'date': datetime.fromtimestamp(item['dt']),
-                        'temperature': item['main']['temp'],
-                        'humidity': item['main']['humidity'],
-                        'rainfall': item.get('rain', {}).get('3h', 0) * 8,
+                        'date': datetime.combine(day_key, datetime.min.time()),
+                        'temperature': np.mean(day_data['temps']),
+                        'temp_min': min(day_data['temps']),
+                        'temp_max': max(day_data['temps']),
+                        'humidity': np.mean(day_data['humidity']),
+                        'rainfall': day_data['rainfall'],
                         'type': 'forecast',
-                        'description': item['weather'][0]['description'],
-                        'temp_min': item['main']['temp_min'],
-                        'temp_max': item['main']['temp_max'],
-                        'wind_speed': item['wind']['speed'],
-                        'clouds': item['clouds']['all']
+                        'description': max(day_data['descriptions'], key=day_data['descriptions'].count)
                     }
                     forecast_data.append(forecast)
 
-                # Generate extended forecast with patterns
-                last_date = max(d['date'] for d in forecast_data)
-                for day in range(1, forecast_days - 5):
-                    base_forecast = forecast_data[-1]
-                    
-                    for hour in range(24):
-                        date = last_date + timedelta(days=day, hours=hour)
-                        
-                        # Add seasonal trend
-                        seasonal_factor = np.sin(2 * np.pi * (date.timetuple().tm_yday / 365))
-                        
-                        # Temperature with daily and seasonal patterns
-                        temp = base_forecast['temperature'] + \
-                               3 * np.sin((hour - 6) * np.pi / 12) + \
-                               2 * seasonal_factor + \
-                               np.random.normal(0, 1)
-                        
-                        # Humidity with inverse pattern
-                        humidity = base_forecast['humidity'] - \
-                                 10 * np.sin((hour - 6) * np.pi / 12) - \
-                                 5 * seasonal_factor + \
-                                 np.random.normal(0, 5)
-                        
-                        # Rainfall with seasonal influence
-                        rain_chance = 0.1 + 0.2 * np.sin((hour - 12) * np.pi / 12) + 0.1 * seasonal_factor
-                        rainfall = np.random.exponential(base_rainfall) if np.random.random() < rain_chance else 0
-
-                        extended_forecast = {
-                            'date': date,
-                            'temperature': temp,
-                            'humidity': np.clip(humidity, 0, 100),
-                            'rainfall': rainfall,
-                            'type': 'forecast_extended',
-                            'description': self.get_weather_description(temp, humidity, rainfall),
-                            'temp_min': temp - np.random.uniform(0, 2),
-                            'temp_max': temp + np.random.uniform(0, 2),
-                            'wind_speed': base_forecast['wind_speed'] + np.random.normal(0, 1),
-                            'clouds': np.clip(base_forecast['clouds'] + np.random.normal(0, 10), 0, 100)
-                        }
-                        forecast_data.append(extended_forecast)
-                    
         except Exception as e:
             print(f"Error fetching forecast data: {e}")
 
-        # Combine and process data
+        # Combine all data
         all_data = pd.DataFrame(historical_data + forecast_data)
         
         if not all_data.empty:
-            # Sort and clean data
+            # Ensure numeric types
+            numeric_columns = ['temperature', 'humidity', 'rainfall', 'temp_min', 'temp_max']
+            for col in numeric_columns:
+                all_data[col] = pd.to_numeric(all_data[col], errors='coerce')
+
+            # Sort by date
             all_data = all_data.sort_values('date')
             
-            # Resample to hourly data while preserving patterns
-            all_data = all_data.set_index('date').resample('1H').mean().reset_index()
-            
-            # Add analysis columns
+            # Add additional columns
             all_data['month'] = all_data['date'].dt.month
             all_data['season'] = all_data['month'].map(self.tanzania_seasons)
             
             # Calculate rolling averages
-            all_data['temp_7day_avg'] = all_data['temperature'].rolling(window=168, min_periods=1).mean()  # 7 days * 24 hours
-            all_data['humidity_7day_avg'] = all_data['humidity'].rolling(window=168, min_periods=1).mean()
-            all_data['rainfall_7day_avg'] = all_data['rainfall'].rolling(window=168, min_periods=1).mean()
+            all_data['temp_7day_avg'] = all_data['temperature'].rolling(window=7, min_periods=1).mean()
+            all_data['humidity_7day_avg'] = all_data['humidity'].rolling(window=7, min_periods=1).mean()
+            all_data['rainfall_7day_avg'] = all_data['rainfall'].rolling(window=7, min_periods=1).mean()
+            
+            # Calculate daily suitability
+            all_data['daily_suitability'] = self.calculate_daily_suitability(all_data)
             
-            # Fill missing values
-            all_data = all_data.fillna(method='ffill').fillna(method='bfill')
+            # Calculate NDVI
+            all_data['estimated_ndvi'] = self.estimate_ndvi(all_data)
+
+            # Group by date to get daily values while preserving types
+            daily_data = pd.DataFrame()
+            daily_data['date'] = all_data['date'].dt.date.unique()
+            daily_data = daily_data.set_index('date')
             
-            # Calculate daily aggregates
-            daily_data = all_data.groupby(all_data['date'].dt.date).agg({
-                'temperature': ['mean', 'min', 'max'],
+            # Aggregate numeric columns
+            numeric_aggs = {
+                'temperature': 'mean',
                 'humidity': 'mean',
                 'rainfall': 'sum',
+                'temp_min': 'min',
+                'temp_max': 'max',
+                'temp_7day_avg': 'last',
+                'humidity_7day_avg': 'last',
+                'rainfall_7day_avg': 'last',
+                'daily_suitability': 'mean',
+                'estimated_ndvi': 'mean'
+            }
+            
+            # Aggregate categoric columns
+            categoric_aggs = {
                 'type': 'first',
                 'description': 'first',
-                'wind_speed': 'mean',
-                'clouds': 'mean',
                 'season': 'first'
-            }).reset_index()
-
-            # Flatten column names
-            daily_data.columns = ['date', 'temperature', 'temp_min', 'temp_max', 'humidity', 
-                                'rainfall', 'type', 'description', 'wind_speed', 'clouds', 'season']
-            
-            # Convert date back to datetime
-            daily_data['date'] = pd.to_datetime(daily_data['date'])
+            }
             
-            # Add suitability and NDVI calculations
-            daily_data['daily_suitability'] = self.calculate_daily_suitability(daily_data)
-            daily_data['estimated_ndvi'] = self.estimate_ndvi(daily_data)
+            # Perform aggregations separately
+            numeric_data = all_data.groupby(all_data['date'].dt.date).agg(numeric_aggs)
+            categoric_data = all_data.groupby(all_data['date'].dt.date).agg(categoric_aggs)
             
+            # Combine the results
+            daily_data = pd.concat([numeric_data, categoric_data], axis=1)
+            daily_data = daily_data.reset_index()
+            daily_data['date'] = pd.to_datetime(daily_data['date'])
+
             return daily_data
 
         return pd.DataFrame()