From 6816c4a55190ebb7ad9986f2ba74109f0d44b9a2 Mon Sep 17 00:00:00 2001 From: Khalim Conn-Kowlessar Date: Thu, 5 Oct 2023 14:54:10 +0100 Subject: [PATCH] refactoring repo wip --- data_collection/README.md | 5 - data_collection/adzuna.py | 86 --- data_collection/config.py | 7 - data_collection/data/.DS_Store | Bin 6148 -> 0 bytes ..._Names_and_Codes_in_the_United_Kingdom.csv | 651 ------------------ data_collection/local_authority.py | 1 - data_collection/requirements.txt | 4 - .../land_registry}/__init__.py | 0 {land_registry => etl/land_registry}/app.py | 0 .../land_registry}/sample_addresses.pkl | Bin 10 files changed, 754 deletions(-) delete mode 100644 data_collection/README.md delete mode 100644 data_collection/adzuna.py delete mode 100644 data_collection/config.py delete mode 100644 data_collection/data/.DS_Store delete mode 100644 data_collection/data/Westminster_Parliamentary_Constituencies_(December_2022)_Names_and_Codes_in_the_United_Kingdom.csv delete mode 100644 data_collection/local_authority.py delete mode 100644 data_collection/requirements.txt rename {land_registry => etl/land_registry}/__init__.py (100%) rename {land_registry => etl/land_registry}/app.py (100%) rename {land_registry => etl/land_registry}/sample_addresses.pkl (100%) diff --git a/data_collection/README.md b/data_collection/README.md deleted file mode 100644 index 1e42378d..00000000 --- a/data_collection/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Data Collection - -This service is specifically focused on the collection of data external sources which aren't easily -accessed via api or via downloadable data sources. For example, wages data requires a specific application to -pull that data from websites, e.g. from Adzuna's api \ No newline at end of file diff --git a/data_collection/adzuna.py b/data_collection/adzuna.py deleted file mode 100644 index d053a2bb..00000000 --- a/data_collection/adzuna.py +++ /dev/null @@ -1,86 +0,0 @@ -import requests -import json -from data_collection.config import ADZUNA_API_KEY, ADZUNA_APP_ID - -import pandas as pd -import os -import time -from tqdm import tqdm - -""" -Table of constituencies and their codes can be downloaded from the Office of National Statistics, found here: -https://geoportal.statistics.gov.uk/datasets/ons::westminster-parliamentary-constituencies-december-2022-names-and --codes-in-the-united-kingdom/explore -""" - -constituencies = pd.read_csv( - os.path.abspath( - os.path.dirname( - __file__)) + "/data_collection/data/Westminster_Parliamentary_Constituencies_(" - "December_2022)_Names_and_Codes_in_the_United_Kingdom.csv" -) - -constituencies["location_type"] = "constituency" - - -def retry_api_call(job_title, location, max_retries=10): - for i in range(max_retries): - try: - response = get_adzuna_jobs(job_title, location) - return response - except (requests.HTTPError, requests.ConnectionError): - print(f"Attempt {i + 1} failed. Retrying in 2 seconds...") - time.sleep(2) - print(f"Failed after {max_retries} attempts.") - return None - - -def get_adzuna_jobs(job_title, location): - base_url = "https://api.adzuna.com/v1/api/jobs" - country_code = "gb" - - url = f"{base_url}/{country_code}/search/1" - - params = { - "app_id": ADZUNA_APP_ID, - "app_key": ADZUNA_API_KEY, - "results_per_page": 25, - "what": job_title, - "where": location, - "content-type": "application/json", - "distance": 10 - } - - response = requests.get(url, params=params) - response.raise_for_status() - - jobs = json.loads(response.text) - return jobs - - -JOB_TITLES = [ - "insulation installer", "internal wall insulation installer", "external wall insulation installer", - "cavity wall insulation installer", "loft insulation installer", "roof insulation installer", - "spray foam insulation installer", "insulation technician", "insulation engineer", "iwi insulation installer", - "iwi installer", "ewi insulation installer", "ewi installer", "cwi insulation installer", "cwi installer", -] - -results = [] -for i, job_title in enumerate(JOB_TITLES): - print("Pulling job title %s of %s" % (str(i + 1), str(len(JOB_TITLES)))) - for _, location_config in tqdm(constituencies.iterrows(), total=constituencies.shape[0]): - - location = location_config["PCON22NM"] - jobs = retry_api_call(job_title, location) - time.sleep(0.5) - if jobs["results"]: - for job in jobs['results']: - to_append = { - "job_title": job_title, - "search_location": location, - "search_location_code": location_config["PCON22CD"], - **job - } - results.append(to_append) - -results_df = pd.DataFrame(results) diff --git a/data_collection/config.py b/data_collection/config.py deleted file mode 100644 index b2d5a363..00000000 --- a/data_collection/config.py +++ /dev/null @@ -1,7 +0,0 @@ -import os -from dotenv import load_dotenv - -load_dotenv(dotenv_path='data_collection/.env') - -ADZUNA_API_KEY = os.environ.get('ADZUNA_API_KEY') -ADZUNA_APP_ID = os.environ.get('ADZUNA_APP_ID') diff --git a/data_collection/data/.DS_Store b/data_collection/data/.DS_Store deleted file mode 100644 index 469b1421e3b455f9d02a34ade3a4d02b362d24b0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKJxc>o5S-N%0h>S?%l8wEzu-tNl@JB7Ry1WGh1g%^@A9W*_M*p2vi zyko`83mzDE*z7+)4)4d~!6`B+AO)m=6p#W^;7kRok7W;NOQvckBv> z#Q1bD#0Wr~F&)Nr%o4=r31U|`Br-#@q!N>AHDXxOnQv9s6%L6>ht=?5^<=9F#p3C_ zzePE$OH`BsQedpWb#52l|F`r%=KnECJ1HOqE=mEL_0Rp9uT;Ht@^ap58~u&$H9vGW pu7ko5?U)$tm>X}$w^5XJ&DT8d3Wvm?Gaq!Keg<3@nH2bM1x_0w79Icq diff --git a/data_collection/data/Westminster_Parliamentary_Constituencies_(December_2022)_Names_and_Codes_in_the_United_Kingdom.csv b/data_collection/data/Westminster_Parliamentary_Constituencies_(December_2022)_Names_and_Codes_in_the_United_Kingdom.csv deleted file mode 100644 index d0158806..00000000 --- a/data_collection/data/Westminster_Parliamentary_Constituencies_(December_2022)_Names_and_Codes_in_the_United_Kingdom.csv +++ /dev/null @@ -1,651 +0,0 @@ -PCON22CD,PCON22NM,ObjectId -E14000530,Aldershot,1 -E14000531,Aldridge-Brownhills,2 -E14000532,Altrincham and Sale West,3 -E14000533,Amber Valley,4 -E14000534,Arundel and South Downs,5 -E14000535,Ashfield,6 -E14000536,Ashford,7 -E14000537,Ashton-under-Lyne,8 -E14000538,Aylesbury,9 -E14000539,Banbury,10 -E14000540,Barking,11 -E14000541,Barnsley Central,12 -E14000542,Barnsley East,13 -E14000543,Barrow and Furness,14 -E14000544,Basildon and Billericay,15 -E14000545,Basingstoke,16 -E14000546,Bassetlaw,17 -E14000547,Bath,18 -E14000548,Batley and Spen,19 -E14000549,Battersea,20 -E14000550,Beaconsfield,21 -E14000551,Beckenham,22 -E14000552,Bedford,23 -E14000553,Bermondsey and Old Southwark,24 -E14000554,Berwick-upon-Tweed,25 -E14000555,Bethnal Green and Bow,26 -E14000556,Beverley and Holderness,27 -E14000557,Bexhill and Battle,28 -E14000558,Bexleyheath and Crayford,29 -E14000559,Birkenhead,30 -E14000560,"Birmingham, Edgbaston",31 -E14000561,"Birmingham, Erdington",32 -E14000562,"Birmingham, Hall Green",33 -E14000563,"Birmingham, Hodge Hill",34 -E14000564,"Birmingham, Ladywood",35 -E14000565,"Birmingham, Northfield",36 -E14000566,"Birmingham, Perry Barr",37 -E14000567,"Birmingham, Selly Oak",38 -E14000568,"Birmingham, Yardley",39 -E14000569,Bishop Auckland,40 -E14000570,Blackburn,41 -E14000571,Blackley and Broughton,42 -E14000572,Blackpool North and Cleveleys,43 -E14000573,Blackpool South,44 -E14000574,Blaydon,45 -E14000575,Blyth Valley,46 -E14000576,Bognor Regis and Littlehampton,47 -E14000577,Bolsover,48 -E14000578,Bolton North East,49 -E14000579,Bolton South East,50 -E14000830,Newbury,51 -E14000831,Newcastle upon Tyne Central,52 -E14000832,Newcastle upon Tyne East,53 -E14000833,Newcastle upon Tyne North,54 -E14000834,Newcastle-under-Lyme,55 -E14000835,Newton Abbot,56 -E14000836,"Normanton, Pontefract and Castleford",57 -E14000837,North Cornwall,58 -E14000838,North Devon,59 -E14000839,North Dorset,60 -E14000840,North Durham,61 -E14000841,North East Bedfordshire,62 -E14000842,North East Cambridgeshire,63 -E14000843,North East Derbyshire,64 -E14000844,North East Hampshire,65 -E14000845,North East Hertfordshire,66 -E14000846,North East Somerset,67 -E14000847,North Herefordshire,68 -E14000848,North Norfolk,69 -E14000849,North Shropshire,70 -E14000850,North Somerset,71 -E14000851,North Swindon,72 -E14000852,North Thanet,73 -E14000853,North Tyneside,74 -E14000854,North Warwickshire,75 -E14000855,North West Cambridgeshire,76 -E14000856,North West Durham,77 -E14000857,North West Hampshire,78 -E14000858,North West Leicestershire,79 -E14000859,North West Norfolk,80 -E14000860,North Wiltshire,81 -E14000861,Northampton North,82 -E14000862,Northampton South,83 -E14000863,Norwich North,84 -E14000864,Norwich South,85 -E14000865,Nottingham East,86 -E14000866,Nottingham North,87 -E14000867,Nottingham South,88 -E14000868,Nuneaton,89 -E14000869,Old Bexley and Sidcup,90 -E14000870,Oldham East and Saddleworth,91 -E14000871,Oldham West and Royton,92 -E14000872,Orpington,93 -E14000873,Oxford East,94 -E14000874,Oxford West and Abingdon,95 -E14000875,Pendle,96 -E14000876,Penistone and Stocksbridge,97 -E14000877,Penrith and The Border,98 -E14000878,Peterborough,99 -E14000879,"Plymouth, Moor View",100 -E14000580,Bolton West,101 -E14000581,Bootle,102 -E14000582,Boston and Skegness,103 -E14000583,Bosworth,104 -E14000584,Bournemouth East,105 -E14000585,Bournemouth West,106 -E14000586,Bracknell,107 -E14000587,Bradford East,108 -E14000588,Bradford South,109 -E14000589,Bradford West,110 -E14000590,Braintree,111 -E14000591,Brent Central,112 -E14000592,Brent North,113 -E14000593,Brentford and Isleworth,114 -E14000594,Brentwood and Ongar,115 -E14000595,Bridgwater and West Somerset,116 -E14000596,Brigg and Goole,117 -E14000597,"Brighton, Kemptown",118 -E14000598,"Brighton, Pavilion",119 -E14000599,Bristol East,120 -E14000600,Bristol North West,121 -E14000601,Bristol South,122 -E14000602,Bristol West,123 -E14000603,Broadland,124 -E14000604,Bromley and Chislehurst,125 -E14000605,Bromsgrove,126 -E14000606,Broxbourne,127 -E14000607,Broxtowe,128 -E14000608,Buckingham,129 -E14000609,Burnley,130 -E14000610,Burton,131 -E14000611,Bury North,132 -E14000612,Bury South,133 -E14000613,Bury St Edmunds,134 -E14000614,Calder Valley,135 -E14000615,Camberwell and Peckham,136 -E14000616,Camborne and Redruth,137 -E14000617,Cambridge,138 -E14000618,Cannock Chase,139 -E14000619,Canterbury,140 -E14000620,Carlisle,141 -E14000621,Carshalton and Wallington,142 -E14000622,Castle Point,143 -E14000623,Central Devon,144 -E14000624,Central Suffolk and North Ipswich,145 -E14000625,Charnwood,146 -E14000626,Chatham and Aylesford,147 -E14000627,Cheadle,148 -E14000628,Chelmsford,149 -E14000629,Chelsea and Fulham,150 -E14000630,Cheltenham,151 -E14000631,Chesham and Amersham,152 -E14000632,Chesterfield,153 -E14000633,Chichester,154 -E14000634,Chingford and Woodford Green,155 -E14000635,Chippenham,156 -E14000636,Chipping Barnet,157 -E14000637,Chorley,158 -E14000638,Christchurch,159 -E14000639,Cities of London and Westminster,160 -E14000640,City of Chester,161 -E14000641,City of Durham,162 -E14000642,Clacton,163 -E14000643,Cleethorpes,164 -E14000644,Colchester,165 -E14000645,Colne Valley,166 -E14000646,Congleton,167 -E14000647,Copeland,168 -E14000648,Corby,169 -E14000649,Coventry North East,170 -E14000650,Coventry North West,171 -E14000651,Coventry South,172 -E14000652,Crawley,173 -E14000653,Crewe and Nantwich,174 -E14000654,Croydon Central,175 -E14000655,Croydon North,176 -E14000656,Croydon South,177 -E14000657,Dagenham and Rainham,178 -E14000658,Darlington,179 -E14000659,Dartford,180 -E14000660,Daventry,181 -E14000661,Denton and Reddish,182 -E14000662,Derby North,183 -E14000663,Derby South,184 -E14000664,Derbyshire Dales,185 -E14000665,Devizes,186 -E14000666,Dewsbury,187 -E14000667,Don Valley,188 -E14000668,Doncaster Central,189 -E14000669,Doncaster North,190 -E14000670,Dover,191 -E14000671,Dudley North,192 -E14000672,Dudley South,193 -E14000673,Dulwich and West Norwood,194 -E14000674,Ealing Central and Acton,195 -E14000675,Ealing North,196 -E14000676,"Ealing, Southall",197 -E14000677,Easington,198 -E14000678,East Devon,199 -E14000679,East Ham,200 -E14000780,Leeds North West,201 -E14000781,Leeds West,202 -E14000782,Leicester East,203 -E14000783,Leicester South,204 -E14000784,Leicester West,205 -E14000785,Leigh,206 -E14000786,Lewes,207 -E14000787,Lewisham East,208 -E14000788,Lewisham West and Penge,209 -E14000789,"Lewisham, Deptford",210 -E14000790,Leyton and Wanstead,211 -E14000791,Lichfield,212 -E14000792,Lincoln,213 -E14000793,"Liverpool, Riverside",214 -E14000794,"Liverpool, Walton",215 -E14000795,"Liverpool, Wavertree",216 -E14000796,"Liverpool, West Derby",217 -E14000797,Loughborough,218 -E14000798,Louth and Horncastle,219 -E14000799,Ludlow,220 -E14000800,Luton North,221 -E14000801,Luton South,222 -E14000802,Macclesfield,223 -E14000803,Maidenhead,224 -E14000804,Maidstone and The Weald,225 -E14000805,Makerfield,226 -E14000806,Maldon,227 -E14000807,Manchester Central,228 -E14000808,"Manchester, Gorton",229 -E14000809,"Manchester, Withington",230 -E14000810,Mansfield,231 -E14000811,Meon Valley,232 -E14000812,Meriden,233 -E14000813,Mid Bedfordshire,234 -E14000814,Mid Derbyshire,235 -E14000815,Mid Dorset and North Poole,236 -E14000816,Mid Norfolk,237 -E14000817,Mid Sussex,238 -E14000818,Mid Worcestershire,239 -E14000819,Middlesbrough,240 -E14000820,Middlesbrough South and East Cleveland,241 -E14000821,Milton Keynes North,242 -E14000822,Milton Keynes South,243 -E14000823,Mitcham and Morden,244 -E14000824,Mole Valley,245 -E14000825,Morecambe and Lunesdale,246 -E14000826,Morley and Outwood,247 -E14000827,New Forest East,248 -E14000828,New Forest West,249 -E14000829,Newark,250 -E14000680,East Hampshire,251 -E14000681,East Surrey,252 -E14000682,East Worthing and Shoreham,253 -E14000683,East Yorkshire,254 -E14000880,"Plymouth, Sutton and Devonport",255 -E14000684,Eastbourne,256 -E14000685,Eastleigh,257 -E14000881,Poole,258 -E14000686,Eddisbury,259 -E14000882,Poplar and Limehouse,260 -E14000687,Edmonton,261 -E14000883,Portsmouth North,262 -E14000688,Ellesmere Port and Neston,263 -E14000884,Portsmouth South,264 -E14000689,Elmet and Rothwell,265 -E14000885,Preston,266 -E14000690,Eltham,267 -E14000886,Pudsey,268 -E14000691,Enfield North,269 -E14000887,Putney,270 -E14000692,"Enfield, Southgate",271 -E14000888,Rayleigh and Wickford,272 -E14000693,Epping Forest,273 -E14000889,Reading East,274 -E14000694,Epsom and Ewell,275 -E14000890,Reading West,276 -E14000695,Erewash,277 -E14000891,Redcar,278 -E14000696,Erith and Thamesmead,279 -E14000892,Redditch,280 -E14000697,Esher and Walton,281 -E14000893,Reigate,282 -E14000698,Exeter,283 -E14000894,Ribble Valley,284 -E14000699,Fareham,285 -E14000895,Richmond (Yorks),286 -E14000700,Faversham and Mid Kent,287 -E14000896,Richmond Park,288 -E14000701,Feltham and Heston,289 -E14000897,Rochdale,290 -E14000702,Filton and Bradley Stoke,291 -E14000898,Rochester and Strood,292 -E14000703,Finchley and Golders Green,293 -E14000899,Rochford and Southend East,294 -E14000704,Folkestone and Hythe,295 -E14000900,Romford,296 -E14000705,Forest of Dean,297 -E14000901,Romsey and Southampton North,298 -E14000706,Fylde,299 -E14000902,Rossendale and Darwen,300 -E14000707,Gainsborough,301 -E14000903,Rother Valley,302 -E14000904,Rotherham,303 -E14000905,Rugby,304 -E14000906,"Ruislip, Northwood and Pinner",305 -E14000907,Runnymede and Weybridge,306 -E14000908,Rushcliffe,307 -E14000909,Rutland and Melton,308 -E14000910,Saffron Walden,309 -E14000911,Salford and Eccles,310 -E14000912,Salisbury,311 -E14000913,Scarborough and Whitby,312 -E14000914,Scunthorpe,313 -E14000915,Sedgefield,314 -E14000916,Sefton Central,315 -E14000917,Selby and Ainsty,316 -E14000918,Sevenoaks,317 -E14000919,Sheffield Central,318 -E14000920,Sheffield South East,319 -E14000921,"Sheffield, Brightside and Hillsborough",320 -E14000922,"Sheffield, Hallam",321 -E14000923,"Sheffield, Heeley",322 -E14000924,Sherwood,323 -E14000925,Shipley,324 -E14000926,Shrewsbury and Atcham,325 -E14000927,Sittingbourne and Sheppey,326 -E14000928,Skipton and Ripon,327 -E14000929,Sleaford and North Hykeham,328 -E14000730,Harrogate and Knaresborough,329 -E14000731,Harrow East,330 -E14000732,Harrow West,331 -E14000733,Hartlepool,332 -E14000734,Harwich and North Essex,333 -E14000735,Hastings and Rye,334 -E14000736,Havant,335 -E14000737,Hayes and Harlington,336 -E14000738,Hazel Grove,337 -E14000739,Hemel Hempstead,338 -E14000740,Hemsworth,339 -E14000741,Hendon,340 -E14000742,Henley,341 -E14000743,Hereford and South Herefordshire,342 -E14000744,Hertford and Stortford,343 -E14000745,Hertsmere,344 -E14000746,Hexham,345 -E14000747,Heywood and Middleton,346 -E14000748,High Peak,347 -E14000749,Hitchin and Harpenden,348 -E14000750,Holborn and St Pancras,349 -E14000751,Hornchurch and Upminster,350 -E14000752,Hornsey and Wood Green,351 -E14000753,Horsham,352 -E14000754,Houghton and Sunderland South,353 -E14000755,Hove,354 -E14000756,Huddersfield,355 -E14000757,Huntingdon,356 -E14000758,Hyndburn,357 -E14000759,Ilford North,358 -E14000760,Ilford South,359 -E14000761,Ipswich,360 -E14000762,Isle of Wight,361 -E14000763,Islington North,362 -E14000764,Islington South and Finsbury,363 -E14000765,Jarrow,364 -E14000766,Keighley,365 -E14000767,Kenilworth and Southam,366 -E14000768,Kensington,367 -E14000769,Kettering,368 -E14000770,Kingston and Surbiton,369 -E14000771,Kingston upon Hull East,370 -E14000772,Kingston upon Hull North,371 -E14000773,Kingston upon Hull West and Hessle,372 -E14000774,Kingswood,373 -E14000775,Knowsley,374 -E14000776,Lancaster and Fleetwood,375 -E14000777,Leeds Central,376 -E14000778,Leeds East,377 -E14000779,Leeds North East,378 -E14000708,Garston and Halewood,379 -E14000709,Gateshead,380 -E14000710,Gedling,381 -E14000711,Gillingham and Rainham,382 -E14000712,Gloucester,383 -E14000713,Gosport,384 -E14000714,Grantham and Stamford,385 -E14000715,Gravesham,386 -E14000716,Great Grimsby,387 -E14000717,Great Yarmouth,388 -E14000718,Greenwich and Woolwich,389 -E14000719,Guildford,390 -E14000720,Hackney North and Stoke Newington,391 -E14000721,Hackney South and Shoreditch,392 -E14000722,Halesowen and Rowley Regis,393 -E14000723,Halifax,394 -E14000724,Haltemprice and Howden,395 -E14000725,Halton,396 -E14000726,Hammersmith,397 -E14000727,Hampstead and Kilburn,398 -E14000728,Harborough,399 -E14000729,Harlow,400 -E14000930,Slough,401 -E14000931,Solihull,402 -E14000932,Somerton and Frome,403 -E14000933,South Basildon and East Thurrock,404 -E14000934,South Cambridgeshire,405 -E14000935,South Derbyshire,406 -E14000936,South Dorset,407 -E14000937,South East Cambridgeshire,408 -E14000938,South East Cornwall,409 -E14000939,South Holland and The Deepings,410 -E14000940,South Leicestershire,411 -E14000941,South Norfolk,412 -E14000942,South Northamptonshire,413 -E14000943,South Ribble,414 -E14000944,South Shields,415 -E14000945,South Staffordshire,416 -E14000946,South Suffolk,417 -E14000947,South Swindon,418 -E14000948,South Thanet,419 -E14000949,South West Bedfordshire,420 -E14000950,South West Devon,421 -E14000951,South West Hertfordshire,422 -E14000952,South West Norfolk,423 -E14000953,South West Surrey,424 -E14000954,South West Wiltshire,425 -E14000955,"Southampton, Itchen",426 -E14000956,"Southampton, Test",427 -E14000957,Southend West,428 -E14000958,Southport,429 -E14000959,Spelthorne,430 -E14000960,St Albans,431 -E14000961,St Austell and Newquay,432 -E14000962,St Helens North,433 -E14000963,St Helens South and Whiston,434 -E14000964,St Ives,435 -E14000965,Stafford,436 -E14000966,Staffordshire Moorlands,437 -E14000967,Stalybridge and Hyde,438 -E14000968,Stevenage,439 -E14000969,Stockport,440 -E14000970,Stockton North,441 -E14000971,Stockton South,442 -E14000972,Stoke-on-Trent Central,443 -E14000973,Stoke-on-Trent North,444 -E14000974,Stoke-on-Trent South,445 -E14000975,Stone,446 -E14000976,Stourbridge,447 -E14000977,Stratford-on-Avon,448 -E14000978,Streatham,449 -E14000979,Stretford and Urmston,450 -E14000980,Stroud,451 -E14000981,Suffolk Coastal,452 -E14000982,Sunderland Central,453 -E14000983,Surrey Heath,454 -E14000984,Sutton and Cheam,455 -E14000985,Sutton Coldfield,456 -E14000986,Tamworth,457 -E14000987,Tatton,458 -E14000988,Taunton Deane,459 -E14000989,Telford,460 -E14000990,Tewkesbury,461 -E14000991,The Cotswolds,462 -E14000992,The Wrekin,463 -E14000993,Thirsk and Malton,464 -E14000994,Thornbury and Yate,465 -E14000995,Thurrock,466 -E14000996,Tiverton and Honiton,467 -E14000997,Tonbridge and Malling,468 -E14000998,Tooting,469 -E14000999,Torbay,470 -E14001000,Torridge and West Devon,471 -E14001001,Totnes,472 -E14001002,Tottenham,473 -E14001003,Truro and Falmouth,474 -E14001004,Tunbridge Wells,475 -E14001005,Twickenham,476 -E14001006,Tynemouth,477 -E14001007,Uxbridge and South Ruislip,478 -E14001008,Vauxhall,479 -E14001009,Wakefield,480 -E14001010,Wallasey,481 -E14001011,Walsall North,482 -E14001012,Walsall South,483 -E14001013,Walthamstow,484 -E14001014,Wansbeck,485 -E14001015,Wantage,486 -E14001016,Warley,487 -E14001017,Warrington North,488 -E14001018,Warrington South,489 -E14001019,Warwick and Leamington,490 -E14001020,Washington and Sunderland West,491 -E14001021,Watford,492 -E14001022,Waveney,493 -E14001023,Wealden,494 -E14001024,Weaver Vale,495 -E14001025,Wellingborough,496 -E14001026,Wells,497 -E14001027,Welwyn Hatfield,498 -E14001028,Wentworth and Dearne,499 -E14001029,West Bromwich East,500 -E14001030,West Bromwich West,501 -E14001031,West Dorset,502 -E14001032,West Ham,503 -E14001033,West Lancashire,504 -E14001034,West Suffolk,505 -E14001035,West Worcestershire,506 -E14001036,Westminster North,507 -E14001037,Westmorland and Lonsdale,508 -E14001038,Weston-Super-Mare,509 -E14001039,Wigan,510 -E14001040,Wimbledon,511 -E14001041,Winchester,512 -E14001042,Windsor,513 -E14001043,Wirral South,514 -E14001044,Wirral West,515 -E14001045,Witham,516 -E14001046,Witney,517 -E14001047,Woking,518 -E14001048,Wokingham,519 -E14001049,Wolverhampton North East,520 -E14001050,Wolverhampton South East,521 -E14001051,Wolverhampton South West,522 -E14001052,Worcester,523 -E14001053,Workington,524 -E14001054,Worsley and Eccles South,525 -E14001055,Worthing West,526 -E14001056,Wycombe,527 -E14001057,Wyre and Preston North,528 -E14001058,Wyre Forest,529 -E14001059,Wythenshawe and Sale East,530 -E14001060,Yeovil,531 -E14001061,York Central,532 -E14001062,York Outer,533 -N06000001,Belfast East,534 -N06000002,Belfast North,535 -N06000003,Belfast South,536 -N06000004,Belfast West,537 -N06000005,East Antrim,538 -N06000006,East Londonderry,539 -N06000007,Fermanagh and South Tyrone,540 -N06000008,Foyle,541 -N06000009,Lagan Valley,542 -N06000010,Mid Ulster,543 -N06000011,Newry and Armagh,544 -N06000012,North Antrim,545 -N06000013,North Down,546 -N06000014,South Antrim,547 -N06000015,South Down,548 -N06000016,Strangford,549 -N06000017,Upper Bann,550 -S14000050,Ochil and South Perthshire,551 -S14000051,Orkney and Shetland,552 -S14000052,Paisley and Renfrewshire North,553 -S14000053,Paisley and Renfrewshire South,554 -S14000054,Perth and North Perthshire,555 -S14000055,"Ross, Skye and Lochaber",556 -S14000056,Rutherglen and Hamilton West,557 -S14000057,Stirling,558 -S14000058,West Aberdeenshire and Kincardine,559 -S14000059,West Dunbartonshire,560 -W07000041,Ynys Môn,561 -W07000042,Delyn,562 -W07000043,Alyn and Deeside,563 -W07000044,Wrexham,564 -W07000045,Llanelli,565 -W07000046,Gower,566 -W07000047,Swansea West,567 -W07000048,Swansea East,568 -W07000049,Aberavon,569 -W07000050,Cardiff Central,570 -W07000051,Cardiff North,571 -W07000052,Rhondda,572 -W07000053,Torfaen,573 -W07000054,Monmouth,574 -W07000055,Newport East,575 -W07000056,Newport West,576 -W07000057,Arfon,577 -W07000058,Aberconwy,578 -W07000059,Clwyd West,579 -W07000060,Vale of Clwyd,580 -W07000061,Dwyfor Meirionnydd,581 -W07000062,Clwyd South,582 -W07000063,Montgomeryshire,583 -W07000064,Ceredigion,584 -W07000065,Preseli Pembrokeshire,585 -W07000066,Carmarthen West and South Pembrokeshire,586 -W07000067,Carmarthen East and Dinefwr,587 -W07000068,Brecon and Radnorshire,588 -W07000069,Neath,589 -W07000070,Cynon Valley,590 -W07000071,Merthyr Tydfil and Rhymney,591 -W07000072,Blaenau Gwent,592 -W07000073,Bridgend,593 -W07000074,Ogmore,594 -W07000075,Pontypridd,595 -W07000076,Caerphilly,596 -W07000077,Islwyn,597 -W07000078,Vale of Glamorgan,598 -W07000079,Cardiff West,599 -W07000080,Cardiff South and Penarth,600 -N06000018,West Tyrone,601 -S14000001,Aberdeen North,602 -S14000002,Aberdeen South,603 -S14000003,Airdrie and Shotts,604 -S14000004,Angus,605 -S14000005,Argyll and Bute,606 -S14000006,"Ayr, Carrick and Cumnock",607 -S14000007,Banff and Buchan,608 -S14000008,"Berwickshire, Roxburgh and Selkirk",609 -S14000009,"Caithness, Sutherland and Easter Ross",610 -S14000010,Central Ayrshire,611 -S14000011,"Coatbridge, Chryston and Bellshill",612 -S14000012,"Cumbernauld, Kilsyth and Kirkintilloch East",613 -S14000013,Dumfries and Galloway,614 -S14000014,"Dumfriesshire, Clydesdale and Tweeddale",615 -S14000015,Dundee East,616 -S14000016,Dundee West,617 -S14000017,Dunfermline and West Fife,618 -S14000018,East Dunbartonshire,619 -S14000019,"East Kilbride, Strathaven and Lesmahagow",620 -S14000020,East Lothian,621 -S14000021,East Renfrewshire,622 -S14000022,Edinburgh East,623 -S14000023,Edinburgh North and Leith,624 -S14000024,Edinburgh South,625 -S14000025,Edinburgh South West,626 -S14000026,Edinburgh West,627 -S14000027,Na h-Eileanan an Iar,628 -S14000028,Falkirk,629 -S14000029,Glasgow Central,630 -S14000030,Glasgow East,631 -S14000031,Glasgow North,632 -S14000032,Glasgow North East,633 -S14000033,Glasgow North West,634 -S14000034,Glasgow South,635 -S14000035,Glasgow South West,636 -S14000036,Glenrothes,637 -S14000037,Gordon,638 -S14000038,Inverclyde,639 -S14000039,"Inverness, Nairn, Badenoch and Strathspey",640 -S14000040,Kilmarnock and Loudoun,641 -S14000041,Kirkcaldy and Cowdenbeath,642 -S14000042,Lanark and Hamilton East,643 -S14000043,Linlithgow and East Falkirk,644 -S14000044,Livingston,645 -S14000045,Midlothian,646 -S14000046,Moray,647 -S14000047,Motherwell and Wishaw,648 -S14000048,North Ayrshire and Arran,649 -S14000049,North East Fife,650 diff --git a/data_collection/local_authority.py b/data_collection/local_authority.py deleted file mode 100644 index 8b137891..00000000 --- a/data_collection/local_authority.py +++ /dev/null @@ -1 +0,0 @@ - diff --git a/data_collection/requirements.txt b/data_collection/requirements.txt deleted file mode 100644 index 12973ef7..00000000 --- a/data_collection/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -requests -python-dotenv -pandas -tqdm \ No newline at end of file diff --git a/land_registry/__init__.py b/etl/land_registry/__init__.py similarity index 100% rename from land_registry/__init__.py rename to etl/land_registry/__init__.py diff --git a/land_registry/app.py b/etl/land_registry/app.py similarity index 100% rename from land_registry/app.py rename to etl/land_registry/app.py diff --git a/land_registry/sample_addresses.pkl b/etl/land_registry/sample_addresses.pkl similarity index 100% rename from land_registry/sample_addresses.pkl rename to etl/land_registry/sample_addresses.pkl