docker working woo

model_data/simulation_system/core/DataLoader.py

@@ -127,7 +127,6 @@ class S3DataLoader:
 
     @staticmethod
     def load(filepath: str, index_col: str | None = None) -> pd.DataFrame:
-
         filepath_split = filepath.split("s3://")[-1].split("/", 1)
         bucket = filepath_split[0]
         key = filepath_split[1]

model_data/simulation_system/handlers/predictions_app.py

@@ -13,35 +13,25 @@ def handler(event, context):
     # Assuming a file in a bucket landing for now?
     # Assuming we have a model to use
 
-    # bucket = event["Records"][0]["s3"]["bucket"]["name"]
-    # key = urllib.parse.unquote_plus(
-    #     event["Records"][0]["s3"]["bucket"]["key"], encoding="utf-8"
-    # )
     payload = event["body"]
     data_path = payload["file_location"]
     property_id = payload["property_id"]
     portfolio_id = payload["portfolio_id"]
     created_at = payload["created_at"]
 
-    # prediction_file = bucket + "/" + key
-
-    # TODO: put a model into s3, both locally and in aws
-    # model_path = os.environ.get("MODEL_PATH", "http://minio:9000/data/model_directory/")
-    model_path = os.environ.get(
-        "MODEL_PATH",
-        f"s3://retrofit-model-directory-{RUNTIME_ENVIRONMENT}/RDSAP_CHANGE/autogluon/rdsap_change-medium_quality-30"
-        "-2023-08-30_11-43-41/deployment/",
-    )
-
     try:
-        outputs = prediction(model_path=model_path, data_path=data_path)
+        # We could fix the model path but for the moment, we just take the best model path based on the registry
+        outputs = prediction(model_path=None, data_path=data_path)
         # Store into s3, with key of {portfolio_id}-{property_id}
-        outputs.to_csv(
-            f"s3://retrofit-sap-prediction-{RUNTIME_ENVIRONMENT}/{portfolio_id}/{property_id}/{created_at}.csv"
-        )
 
-    except (Exception, KeyError, ValueError):
+        storage_filepath = f"s3://retrofit-sap-predictions-{RUNTIME_ENVIRONMENT}/{portfolio_id}/{property_id}/" \
+                           f"{created_at}.csv"
+        outputs.to_csv(storage_filepath)
+        return storage_filepath
+
+    except (Exception, KeyError, ValueError) as e:
         print("Prediction failed")
+        print(e)
 
 
 if __name__ == "__main__":

model_data/simulation_system/predictions.py

@@ -26,6 +26,7 @@ RUNTIME_ENVIRONMENT = os.environ.get("RUNTIME_ENVIRONMENT", "dev")
 
 CLIENT = S3FSClient(runtime_environment=RUNTIME_ENVIRONMENT)
 
+
 # FOR TESTING
 # For now just loading data first and then passing into function (i.e. as if we receive json data and convert to
 # DataFrame)
@@ -142,7 +143,8 @@ def prediction(
 
     logger.info("--- Generating Predictions ---")
     prediction = model.generate_predictions(data=data)
-    return pd.concat([data["id"], prediction], axis=1)
+
+    return pd.concat([pd.Series(data.index, name='id'), prediction], axis=1)
 
     # Save prediction some where?
     # prediction.to_csv("s3?")

model_data/simulation_system/requirements/predictions/predictions.txt

@@ -1,5 +1,6 @@
+boto3
 autogluon==0.8.2
 pandas==1.5.3
-s3fs==2023.6.0
+s3fs
 seaborn==0.12.2
-matplotlib==3.7.2
+matplotlib==3.7.2