343 lines
14 KiB
C#
343 lines
14 KiB
C#
using System.Text;
|
|
using System.Text.Json;
|
|
using Parquet;
|
|
using Parquet.Data;
|
|
using Parquet.Schema;
|
|
|
|
namespace LMEMappeCatastaliApp.Services;
|
|
|
|
public sealed class CadastralCatalogService
|
|
{
|
|
private const string DataBaseUrl = "https://raw.githubusercontent.com/ondata/dati_catastali/main/S_0000_ITALIA/anagrafica/";
|
|
private readonly HttpClient _httpClient = new()
|
|
{
|
|
Timeout = Timeout.InfiniteTimeSpan
|
|
};
|
|
private readonly SemaphoreSlim _downloadLock = new(1, 1);
|
|
private IReadOnlyList<MunicipalityOption>? _municipalities;
|
|
|
|
public CadastralCatalogService()
|
|
{
|
|
_httpClient.DefaultRequestHeaders.UserAgent.ParseAdd("LMEMappeCatastali/0.2");
|
|
}
|
|
|
|
public async Task<IReadOnlyList<RegionOption>> GetRegionsAsync()
|
|
{
|
|
var municipalities = await GetAllMunicipalitiesAsync();
|
|
return municipalities
|
|
.GroupBy(item => item.DataFile, StringComparer.OrdinalIgnoreCase)
|
|
.Select(group => new RegionOption(RegionName(group.Key), group.Key))
|
|
.OrderBy(item => item.Name, StringComparer.CurrentCultureIgnoreCase)
|
|
.ToArray();
|
|
}
|
|
|
|
public async Task<IReadOnlyList<MunicipalityOption>> GetMunicipalitiesAsync(string dataFile)
|
|
{
|
|
var municipalities = await GetAllMunicipalitiesAsync();
|
|
return municipalities
|
|
.Where(item => string.Equals(item.DataFile, dataFile, StringComparison.OrdinalIgnoreCase))
|
|
.OrderBy(item => item.Name, StringComparer.CurrentCultureIgnoreCase)
|
|
.ToArray();
|
|
}
|
|
|
|
public async Task<IReadOnlyList<string>> GetSheetsAsync(
|
|
MunicipalityOption municipality,
|
|
IProgress<CatalogProgress>? progress = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var path = await EnsureRegionFileAsync(municipality.DataFile, progress, cancellationToken);
|
|
progress?.Report(new CatalogProgress("Leggo i fogli del Comune...", null));
|
|
|
|
var sheets = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
await using var reader = await ParquetReader.CreateAsync(path, cancellationToken: cancellationToken);
|
|
var communeField = RequiredField(reader.Schema, "comune");
|
|
var sheetField = RequiredField(reader.Schema, "foglio");
|
|
|
|
for (var groupIndex = 0; groupIndex < reader.RowGroupCount; groupIndex++)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
using var group = reader.OpenRowGroupReader(groupIndex);
|
|
if (!MayContain(group.GetStatistics(communeField), municipality.Code))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var count = checked((int)group.RowCount);
|
|
var communes = new string[count];
|
|
var sheetValues = new string[count];
|
|
await group.ReadAsync(communeField, communes, cancellationToken: cancellationToken);
|
|
await group.ReadAsync(sheetField, sheetValues, cancellationToken: cancellationToken);
|
|
|
|
for (var index = 0; index < count; index++)
|
|
{
|
|
if (string.Equals(communes[index], municipality.Code, StringComparison.OrdinalIgnoreCase)
|
|
&& !string.IsNullOrWhiteSpace(sheetValues[index]))
|
|
{
|
|
sheets.Add(sheetValues[index]);
|
|
}
|
|
}
|
|
}
|
|
|
|
progress?.Report(new CatalogProgress($"Trovati {sheets.Count} fogli.", 1));
|
|
return sheets.OrderBy(value => value, NaturalStringComparer.Instance).ToArray();
|
|
}
|
|
|
|
public async Task<IReadOnlyList<ParcelOption>> GetParcelsAsync(
|
|
MunicipalityOption municipality,
|
|
string sheet,
|
|
IProgress<CatalogProgress>? progress = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
var path = await EnsureRegionFileAsync(municipality.DataFile, progress, cancellationToken);
|
|
progress?.Report(new CatalogProgress("Leggo le particelle del foglio...", null));
|
|
|
|
var parcels = new Dictionary<string, ParcelOption>(StringComparer.OrdinalIgnoreCase);
|
|
await using var reader = await ParquetReader.CreateAsync(path, cancellationToken: cancellationToken);
|
|
var communeField = RequiredField(reader.Schema, "comune");
|
|
var sheetField = RequiredField(reader.Schema, "foglio");
|
|
var parcelField = RequiredField(reader.Schema, "particella");
|
|
var longitudeField = RequiredField(reader.Schema, "x");
|
|
var latitudeField = RequiredField(reader.Schema, "y");
|
|
var inspireIdField = RequiredField(reader.Schema, "INSPIREID_LOCALID");
|
|
|
|
for (var groupIndex = 0; groupIndex < reader.RowGroupCount; groupIndex++)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
using var group = reader.OpenRowGroupReader(groupIndex);
|
|
if (!MayContain(group.GetStatistics(communeField), municipality.Code)
|
|
|| !MayContain(group.GetStatistics(sheetField), sheet))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var count = checked((int)group.RowCount);
|
|
var communes = new string[count];
|
|
var sheets = new string[count];
|
|
await group.ReadAsync(communeField, communes, cancellationToken: cancellationToken);
|
|
await group.ReadAsync(sheetField, sheets, cancellationToken: cancellationToken);
|
|
|
|
var matchingIndexes = Enumerable.Range(0, count)
|
|
.Where(index => string.Equals(communes[index], municipality.Code, StringComparison.OrdinalIgnoreCase)
|
|
&& string.Equals(sheets[index], sheet, StringComparison.OrdinalIgnoreCase))
|
|
.ToArray();
|
|
|
|
if (matchingIndexes.Length == 0)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var parcelValues = new string[count];
|
|
var inspireIds = new string[count];
|
|
var longitudes = new long?[count];
|
|
var latitudes = new long?[count];
|
|
await group.ReadAsync(parcelField, parcelValues, cancellationToken: cancellationToken);
|
|
await group.ReadAsync(inspireIdField, inspireIds, cancellationToken: cancellationToken);
|
|
await group.ReadAsync<long>(longitudeField, longitudes.AsMemory(), cancellationToken: cancellationToken);
|
|
await group.ReadAsync<long>(latitudeField, latitudes.AsMemory(), cancellationToken: cancellationToken);
|
|
|
|
foreach (var index in matchingIndexes)
|
|
{
|
|
var number = parcelValues[index];
|
|
if (string.IsNullOrWhiteSpace(number)
|
|
|| parcels.ContainsKey(number)
|
|
|| !latitudes[index].HasValue
|
|
|| !longitudes[index].HasValue)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
parcels[number] = new ParcelOption(
|
|
number,
|
|
latitudes[index]!.Value / 1_000_000d,
|
|
longitudes[index]!.Value / 1_000_000d,
|
|
inspireIds[index] ?? string.Empty);
|
|
}
|
|
}
|
|
|
|
progress?.Report(new CatalogProgress($"Trovate {parcels.Count} particelle.", 1));
|
|
return parcels.Values.OrderBy(item => item.Number, NaturalStringComparer.Instance).ToArray();
|
|
}
|
|
|
|
private async Task<IReadOnlyList<MunicipalityOption>> GetAllMunicipalitiesAsync()
|
|
{
|
|
if (_municipalities is not null)
|
|
{
|
|
return _municipalities;
|
|
}
|
|
|
|
await using var stream = await FileSystem.Current.OpenAppPackageFileAsync("Data/municipalities.json");
|
|
_municipalities = await JsonSerializer.DeserializeAsync<List<MunicipalityOption>>(
|
|
stream,
|
|
new JsonSerializerOptions { PropertyNameCaseInsensitive = true })
|
|
?? [];
|
|
return _municipalities;
|
|
}
|
|
|
|
private async Task<string> EnsureRegionFileAsync(
|
|
string dataFile,
|
|
IProgress<CatalogProgress>? progress,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var safeFileName = Path.GetFileName(dataFile);
|
|
var destination = Path.Combine(FileSystem.Current.CacheDirectory, $"catasto-{safeFileName}");
|
|
if (IsValidParquetFile(destination))
|
|
{
|
|
progress?.Report(new CatalogProgress("Uso il catalogo regionale salvato sul telefono.", 1));
|
|
return destination;
|
|
}
|
|
|
|
await _downloadLock.WaitAsync(cancellationToken);
|
|
try
|
|
{
|
|
if (IsValidParquetFile(destination))
|
|
{
|
|
return destination;
|
|
}
|
|
|
|
var temporary = destination + ".download";
|
|
if (File.Exists(temporary))
|
|
{
|
|
File.Delete(temporary);
|
|
}
|
|
|
|
using var response = await _httpClient.GetAsync(
|
|
DataBaseUrl + Uri.EscapeDataString(safeFileName),
|
|
HttpCompletionOption.ResponseHeadersRead,
|
|
cancellationToken);
|
|
response.EnsureSuccessStatusCode();
|
|
var total = response.Content.Headers.ContentLength;
|
|
|
|
await using var source = await response.Content.ReadAsStreamAsync(cancellationToken);
|
|
await using var target = new FileStream(temporary, FileMode.CreateNew, FileAccess.Write, FileShare.None, 128 * 1024, true);
|
|
var buffer = new byte[128 * 1024];
|
|
long downloaded = 0;
|
|
int read;
|
|
while ((read = await source.ReadAsync(buffer, cancellationToken)) > 0)
|
|
{
|
|
await target.WriteAsync(buffer.AsMemory(0, read), cancellationToken);
|
|
downloaded += read;
|
|
double? fraction = total > 0 ? (double)downloaded / total.Value : null;
|
|
var megabytes = downloaded / 1_048_576d;
|
|
progress?.Report(new CatalogProgress($"Scarico dati regionali: {megabytes:0.0} MB", fraction));
|
|
}
|
|
|
|
await target.FlushAsync(cancellationToken);
|
|
target.Close();
|
|
File.Move(temporary, destination, true);
|
|
return destination;
|
|
}
|
|
catch
|
|
{
|
|
var temporary = destination + ".download";
|
|
if (File.Exists(temporary))
|
|
{
|
|
File.Delete(temporary);
|
|
}
|
|
throw;
|
|
}
|
|
finally
|
|
{
|
|
_downloadLock.Release();
|
|
}
|
|
}
|
|
|
|
private static bool IsValidParquetFile(string path)
|
|
{
|
|
if (!File.Exists(path) || new FileInfo(path).Length < 12)
|
|
{
|
|
return false;
|
|
}
|
|
|
|
using var stream = File.OpenRead(path);
|
|
Span<byte> marker = stackalloc byte[4];
|
|
if (stream.Read(marker) != marker.Length || Encoding.ASCII.GetString(marker) != "PAR1")
|
|
{
|
|
return false;
|
|
}
|
|
|
|
stream.Seek(-4, SeekOrigin.End);
|
|
return stream.Read(marker) == marker.Length && Encoding.ASCII.GetString(marker) == "PAR1";
|
|
}
|
|
|
|
private static DataField RequiredField(ParquetSchema schema, string name) =>
|
|
schema.FindDataField(name) ?? throw new InvalidDataException($"Colonna catastale mancante: {name}");
|
|
|
|
private static bool MayContain(DataColumnStatistics? statistics, string wanted)
|
|
{
|
|
if (statistics is null)
|
|
{
|
|
return true;
|
|
}
|
|
|
|
var minimum = StatisticText(statistics.MinValue);
|
|
var maximum = StatisticText(statistics.MaxValue);
|
|
return minimum is null || maximum is null
|
|
|| (string.Compare(wanted, minimum, StringComparison.OrdinalIgnoreCase) >= 0
|
|
&& string.Compare(wanted, maximum, StringComparison.OrdinalIgnoreCase) <= 0);
|
|
}
|
|
|
|
private static string? StatisticText(object? value) => value switch
|
|
{
|
|
null => null,
|
|
string text => text,
|
|
byte[] bytes => Encoding.UTF8.GetString(bytes),
|
|
ReadOnlyMemory<byte> memory => Encoding.UTF8.GetString(memory.Span),
|
|
_ => value.ToString()
|
|
};
|
|
|
|
private static string RegionName(string dataFile)
|
|
{
|
|
var key = Path.GetFileNameWithoutExtension(dataFile);
|
|
return key switch
|
|
{
|
|
"01_Piemonte" => "Piemonte",
|
|
"02_ValledAosta" => "Valle d'Aosta",
|
|
"03_Lombardia" => "Lombardia",
|
|
"05_Veneto" => "Veneto",
|
|
"06_Friuli-VeneziaGiulia" => "Friuli-Venezia Giulia",
|
|
"07_Liguria" => "Liguria",
|
|
"08_Emilia-Romagna" => "Emilia-Romagna",
|
|
"09_Toscana" => "Toscana",
|
|
"10_Umbria" => "Umbria",
|
|
"11_Marche" => "Marche",
|
|
"12_Lazio" => "Lazio",
|
|
"13_Abruzzo" => "Abruzzo",
|
|
"14_Molise" => "Molise",
|
|
"15_Campania" => "Campania",
|
|
"16_Puglia" => "Puglia",
|
|
"17_Basilicata" => "Basilicata",
|
|
"18_Calabria" => "Calabria",
|
|
"19_Sicilia" => "Sicilia",
|
|
"20_Sardegna" => "Sardegna",
|
|
_ => key.Length > 3 ? key[3..] : key
|
|
};
|
|
}
|
|
}
|
|
|
|
public sealed record RegionOption(string Name, string DataFile);
|
|
public sealed record MunicipalityOption(string Code, string Name, string DataFile);
|
|
public sealed record ParcelOption(string Number, double Latitude, double Longitude, string InspireId);
|
|
public sealed record CatalogProgress(string Message, double? Fraction);
|
|
|
|
internal sealed class NaturalStringComparer : IComparer<string>
|
|
{
|
|
public static NaturalStringComparer Instance { get; } = new();
|
|
|
|
public int Compare(string? left, string? right)
|
|
{
|
|
if (ReferenceEquals(left, right)) return 0;
|
|
if (left is null) return -1;
|
|
if (right is null) return 1;
|
|
|
|
var leftNumber = long.TryParse(left, out var leftValue);
|
|
var rightNumber = long.TryParse(right, out var rightValue);
|
|
if (leftNumber && rightNumber)
|
|
{
|
|
var numeric = leftValue.CompareTo(rightValue);
|
|
return numeric != 0 ? numeric : string.Compare(left, right, StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
return string.Compare(left, right, StringComparison.CurrentCultureIgnoreCase);
|
|
}
|
|
}
|