more libraries

This commit is contained in:
2024-09-28 22:58:22 -07:00
parent 1973934e95
commit b6e3e24aec
1099 changed files with 326677 additions and 208 deletions

View File

@@ -0,0 +1,19 @@
"""Fiona's command line interface"""
from functools import wraps
def with_context_env(f):
"""Pops the Fiona Env from the passed context and executes the
wrapped func in the context of that obj.
Click's pass_context decorator must precede this decorator, or else
there will be no context in the wrapper args.
"""
@wraps(f)
def wrapper(*args, **kwds):
ctx = args[0]
env = ctx.obj.pop('env')
with env:
return f(*args, **kwds)
return wrapper

View File

@@ -0,0 +1,89 @@
"""$ fio bounds"""
import json
import click
from cligj import precision_opt, use_rs_opt
import fiona
from fiona.fio.helpers import obj_gen
from fiona.fio import with_context_env
from fiona.model import ObjectEncoder
@click.command(short_help="Print the extent of GeoJSON objects")
@precision_opt
@click.option('--explode/--no-explode', default=False,
help="Explode collections into features (default: no).")
@click.option('--with-id/--without-id', default=False,
help="Print GeoJSON ids and bounding boxes together "
"(default: without).")
@click.option('--with-obj/--without-obj', default=False,
help="Print GeoJSON objects and bounding boxes together "
"(default: without).")
@use_rs_opt
@click.pass_context
@with_context_env
def bounds(ctx, precision, explode, with_id, with_obj, use_rs):
"""Print the bounding boxes of GeoJSON objects read from stdin.
Optionally explode collections and print the bounds of their
features.
To print identifiers for input objects along with their bounds
as a {id: identifier, bbox: bounds} JSON object, use --with-id.
To print the input objects themselves along with their bounds
as GeoJSON object, use --with-obj. This has the effect of updating
input objects with {id: identifier, bbox: bounds}.
"""
stdin = click.get_text_stream('stdin')
source = obj_gen(stdin)
for i, obj in enumerate(source):
obj_id = obj.get("id", "collection:" + str(i))
xs = []
ys = []
features = obj.get("features") or [obj]
for j, feat in enumerate(features):
feat_id = feat.get("id", "feature:" + str(i))
w, s, e, n = fiona.bounds(feat)
if precision > 0:
w, s, e, n = (round(v, precision) for v in (w, s, e, n))
if explode:
if with_id:
rec = {"parent": obj_id, "id": feat_id, "bbox": (w, s, e, n)}
elif with_obj:
feat.update(parent=obj_id, bbox=(w, s, e, n))
rec = feat
else:
rec = (w, s, e, n)
if use_rs:
click.echo('\x1e', nl=False)
click.echo(json.dumps(rec, cls=ObjectEncoder))
else:
xs.extend([w, e])
ys.extend([s, n])
if not explode:
w, s, e, n = (min(xs), min(ys), max(xs), max(ys))
if with_id:
rec = {"id": obj_id, "bbox": (w, s, e, n)}
elif with_obj:
obj.update(id=obj_id, bbox=(w, s, e, n))
rec = obj
else:
rec = (w, s, e, n)
if use_rs:
click.echo("\x1e", nl=False)
click.echo(json.dumps(rec, cls=ObjectEncoder))

View File

@@ -0,0 +1,63 @@
import json
import click
from cligj import use_rs_opt
from .helpers import obj_gen, eval_feature_expression
from fiona.fio import with_context_env
from fiona.model import ObjectEncoder
@click.command(short_help="Calculate GeoJSON property by Python expression")
@click.argument('property_name')
@click.argument('expression')
@click.option('--overwrite', is_flag=True, default=False,
help="Overwrite properties, default: False")
@use_rs_opt
@click.pass_context
@with_context_env
def calc(ctx, property_name, expression, overwrite, use_rs):
"""
Create a new property on GeoJSON features using the specified expression.
\b
The expression is evaluated in a restricted namespace containing:
- sum, pow, min, max and the imported math module
- shape (optional, imported from shapely.geometry if available)
- bool, int, str, len, float type conversions
- f (the feature to be evaluated,
allows item access via javascript-style dot notation using munch)
The expression will be evaluated for each feature and its
return value will be added to the properties
as the specified property_name. Existing properties will not
be overwritten by default (an Exception is raised).
Example
\b
$ fio cat data.shp | fio calc sumAB "f.properties.A + f.properties.B"
"""
stdin = click.get_text_stream('stdin')
source = obj_gen(stdin)
for i, obj in enumerate(source):
features = obj.get("features") or [obj]
for j, feat in enumerate(features):
if not overwrite and property_name in feat["properties"]:
raise click.UsageError(
f"{property_name} already exists in properties; "
"rename or use --overwrite"
)
feat["properties"][property_name] = eval_feature_expression(
feat, expression
)
if use_rs:
click.echo("\x1e", nl=False)
click.echo(json.dumps(feat, cls=ObjectEncoder))

View File

@@ -0,0 +1,139 @@
"""fio-cat"""
import json
import warnings
import click
import cligj
import fiona
from fiona.transform import transform_geom
from fiona.model import Feature, ObjectEncoder
from fiona.fio import options, with_context_env
from fiona.fio.helpers import recursive_round
from fiona.errors import AttributeFilterError
warnings.simplefilter("default")
# Cat command
@click.command(short_help="Concatenate and print the features of datasets")
@click.argument("files", nargs=-1, required=True, metavar="INPUTS...")
@click.option(
"--layer",
default=None,
multiple=True,
callback=options.cb_multilayer,
help="Input layer(s), specified as 'fileindex:layer` "
"For example, '1:foo,2:bar' will concatenate layer foo "
"from file 1 and layer bar from file 2",
)
@cligj.precision_opt
@cligj.indent_opt
@cligj.compact_opt
@click.option(
"--ignore-errors/--no-ignore-errors",
default=False,
help="log errors but do not stop serialization.",
)
@options.dst_crs_opt
@cligj.use_rs_opt
@click.option(
"--bbox",
default=None,
metavar="w,s,e,n",
help="filter for features intersecting a bounding box",
)
@click.option(
"--where",
default=None,
help="attribute filter using SQL where clause",
)
@click.option(
"--cut-at-antimeridian",
is_flag=True,
default=False,
help="Optionally cut geometries at the anti-meridian. To be used only for a geographic destination CRS.",
)
@click.option('--where', default=None,
help="attribute filter using SQL where clause")
@options.open_opt
@click.pass_context
@with_context_env
def cat(
ctx,
files,
precision,
indent,
compact,
ignore_errors,
dst_crs,
use_rs,
bbox,
where,
cut_at_antimeridian,
layer,
open_options,
):
"""
Concatenate and print the features of input datasets as a sequence of
GeoJSON features.
When working with a multi-layer dataset the first layer is used by default.
Use the '--layer' option to select a different layer.
"""
dump_kwds = {"sort_keys": True}
if indent:
dump_kwds["indent"] = indent
if compact:
dump_kwds["separators"] = (",", ":")
# Validate file idexes provided in --layer option
# (can't pass the files to option callback)
if layer:
options.validate_multilayer_file_index(files, layer)
# first layer is the default
for i in range(1, len(files) + 1):
if str(i) not in layer.keys():
layer[str(i)] = [0]
try:
if bbox:
try:
bbox = tuple(map(float, bbox.split(",")))
except ValueError:
bbox = json.loads(bbox)
for i, path in enumerate(files, 1):
for lyr in layer[str(i)]:
with fiona.open(path, layer=lyr, **open_options) as src:
for i, feat in src.items(bbox=bbox, where=where):
geom = feat.geometry
if dst_crs:
geom = transform_geom(
src.crs,
dst_crs,
geom,
antimeridian_cutting=cut_at_antimeridian,
)
if precision >= 0:
geom = recursive_round(geom, precision)
feat = Feature(
id=feat.id,
properties=feat.properties,
geometry=geom,
bbox=fiona.bounds(geom),
)
if use_rs:
click.echo("\x1e", nl=False)
click.echo(json.dumps(feat, cls=ObjectEncoder, **dump_kwds))
except AttributeFilterError as e:
raise click.BadParameter("'where' clause is invalid: " + str(e))

View File

@@ -0,0 +1,245 @@
"""fio-collect"""
from functools import partial
import json
import logging
import click
import cligj
from fiona.fio import helpers, options, with_context_env
from fiona.model import Geometry, ObjectEncoder
from fiona.transform import transform_geom
@click.command(short_help="Collect a sequence of features.")
@cligj.precision_opt
@cligj.indent_opt
@cligj.compact_opt
@click.option(
"--record-buffered/--no-record-buffered",
default=False,
help="Economical buffering of writes at record, not collection "
"(default), level.",
)
@click.option(
"--ignore-errors/--no-ignore-errors",
default=False,
help="log errors but do not stop serialization.",
)
@options.src_crs_opt
@click.option(
"--with-ld-context/--without-ld-context",
default=False,
help="add a JSON-LD context to JSON output.",
)
@click.option(
"--add-ld-context-item",
multiple=True,
help="map a term to a URI and add it to the output's JSON LD " "context.",
)
@click.option(
"--parse/--no-parse",
default=True,
help="load and dump the geojson feature (default is True)",
)
@click.pass_context
@with_context_env
def collect(
ctx,
precision,
indent,
compact,
record_buffered,
ignore_errors,
src_crs,
with_ld_context,
add_ld_context_item,
parse,
):
"""Make a GeoJSON feature collection from a sequence of GeoJSON
features and print it."""
logger = logging.getLogger(__name__)
stdin = click.get_text_stream("stdin")
sink = click.get_text_stream("stdout")
dump_kwds = {"sort_keys": True}
if indent:
dump_kwds["indent"] = indent
if compact:
dump_kwds["separators"] = (",", ":")
item_sep = compact and "," or ", "
if src_crs:
if not parse:
raise click.UsageError("Can't specify --src-crs with --no-parse")
transformer = partial(
transform_geom,
src_crs,
"EPSG:4326",
antimeridian_cutting=True,
precision=precision,
)
else:
def transformer(x):
return x
first_line = next(stdin)
# If parsing geojson
if parse:
# If input is RS-delimited JSON sequence.
if first_line.startswith("\x1e"):
def feature_text_gen():
buffer = first_line.strip("\x1e")
for line in stdin:
if line.startswith("\x1e"):
if buffer:
feat = json.loads(buffer)
feat["geometry"] = transformer(
Geometry.from_dict(**feat["geometry"])
)
yield json.dumps(feat, cls=ObjectEncoder, **dump_kwds)
buffer = line.strip("\x1e")
else:
buffer += line
else:
feat = json.loads(buffer)
feat["geometry"] = transformer(
Geometry.from_dict(**feat["geometry"])
)
yield json.dumps(feat, cls=ObjectEncoder, **dump_kwds)
else:
def feature_text_gen():
feat = json.loads(first_line)
feat["geometry"] = transformer(Geometry.from_dict(**feat["geometry"]))
yield json.dumps(feat, cls=ObjectEncoder, **dump_kwds)
for line in stdin:
feat = json.loads(line)
feat["geometry"] = transformer(
Geometry.from_dict(**feat["geometry"])
)
yield json.dumps(feat, cls=ObjectEncoder, **dump_kwds)
# If *not* parsing geojson
else:
# If input is RS-delimited JSON sequence.
if first_line.startswith("\x1e"):
def feature_text_gen():
buffer = first_line.strip("\x1e")
for line in stdin:
if line.startswith("\x1e"):
if buffer:
yield buffer
buffer = line.strip("\x1e")
else:
buffer += line
else:
yield buffer
else:
def feature_text_gen():
yield first_line
yield from stdin
source = feature_text_gen()
if record_buffered:
# Buffer GeoJSON data at the feature level for smaller
# memory footprint.
indented = bool(indent)
rec_indent = "\n" + " " * (2 * (indent or 0))
collection = {"type": "FeatureCollection", "features": []}
if with_ld_context:
collection["@context"] = helpers.make_ld_context(add_ld_context_item)
head, tail = json.dumps(collection, cls=ObjectEncoder, **dump_kwds).split("[]")
sink.write(head)
sink.write("[")
# Try the first record.
try:
i, first = 0, next(source)
if with_ld_context:
first = helpers.id_record(first)
if indented:
sink.write(rec_indent)
sink.write(first.replace("\n", rec_indent))
except StopIteration:
pass
except Exception as exc:
# Ignoring errors is *not* the default.
if ignore_errors:
logger.error(
"failed to serialize file record %d (%s), " "continuing", i, exc
)
else:
# Log error and close up the GeoJSON, leaving it
# more or less valid no matter what happens above.
logger.critical(
"failed to serialize file record %d (%s), " "quitting", i, exc
)
sink.write("]")
sink.write(tail)
if indented:
sink.write("\n")
raise
# Because trailing commas aren't valid in JSON arrays
# we'll write the item separator before each of the
# remaining features.
for i, rec in enumerate(source, 1):
try:
if with_ld_context:
rec = helpers.id_record(rec)
if indented:
sink.write(rec_indent)
sink.write(item_sep)
sink.write(rec.replace("\n", rec_indent))
except Exception as exc:
if ignore_errors:
logger.error(
"failed to serialize file record %d (%s), " "continuing",
i,
exc,
)
else:
logger.critical(
"failed to serialize file record %d (%s), " "quitting",
i,
exc,
)
sink.write("]")
sink.write(tail)
if indented:
sink.write("\n")
raise
# Close up the GeoJSON after writing all features.
sink.write("]")
sink.write(tail)
if indented:
sink.write("\n")
else:
# Buffer GeoJSON data at the collection level. The default.
collection = {"type": "FeatureCollection", "features": []}
if with_ld_context:
collection["@context"] = helpers.make_ld_context(add_ld_context_item)
head, tail = json.dumps(collection, cls=ObjectEncoder, **dump_kwds).split("[]")
sink.write(head)
sink.write("[")
sink.write(",".join(source))
sink.write("]")
sink.write(tail)
sink.write("\n")

View File

@@ -0,0 +1,35 @@
"""$ fio distrib"""
import json
import click
import cligj
from fiona.fio import helpers, with_context_env
from fiona.model import ObjectEncoder
@click.command()
@cligj.use_rs_opt
@click.pass_context
@with_context_env
def distrib(ctx, use_rs):
"""Distribute features from a collection.
Print the features of GeoJSON objects read from stdin.
"""
stdin = click.get_text_stream('stdin')
source = helpers.obj_gen(stdin)
for i, obj in enumerate(source):
obj_id = obj.get("id", "collection:" + str(i))
features = obj.get("features") or [obj]
for j, feat in enumerate(features):
if obj.get("type") == "FeatureCollection":
feat["parent"] = obj_id
feat_id = feat.get("id", "feature:" + str(i))
feat["id"] = feat_id
if use_rs:
click.echo("\x1e", nl=False)
click.echo(json.dumps(feat, cls=ObjectEncoder))

View File

@@ -0,0 +1,198 @@
"""fio-dump"""
from functools import partial
import json
import logging
import click
import cligj
import fiona
from fiona.fio import helpers, options, with_context_env
from fiona.model import Feature, ObjectEncoder
from fiona.transform import transform_geom
@click.command(short_help="Dump a dataset to GeoJSON.")
@click.argument('input', required=True)
@click.option('--layer', metavar="INDEX|NAME", callback=options.cb_layer,
help="Print information about a specific layer. The first "
"layer is used by default. Layers use zero-based "
"numbering when accessed by index.")
@click.option('--encoding', help="Specify encoding of the input file.")
@cligj.precision_opt
@cligj.indent_opt
@cligj.compact_opt
@click.option('--record-buffered/--no-record-buffered', default=False,
help="Economical buffering of writes at record, not collection "
"(default), level.")
@click.option('--ignore-errors/--no-ignore-errors', default=False,
help="log errors but do not stop serialization.")
@click.option('--with-ld-context/--without-ld-context', default=False,
help="add a JSON-LD context to JSON output.")
@click.option('--add-ld-context-item', multiple=True,
help="map a term to a URI and add it to the output's JSON LD "
"context.")
@options.open_opt
@click.pass_context
@with_context_env
def dump(
ctx,
input,
encoding,
precision,
indent,
compact,
record_buffered,
ignore_errors,
with_ld_context,
add_ld_context_item,
layer,
open_options,
):
"""Dump a dataset either as a GeoJSON feature collection (the default)
or a sequence of GeoJSON features."""
logger = logging.getLogger(__name__)
sink = click.get_text_stream('stdout')
dump_kwds = {'sort_keys': True}
if indent:
dump_kwds['indent'] = indent
if compact:
dump_kwds['separators'] = (',', ':')
item_sep = compact and ',' or ', '
if encoding:
open_options["encoding"] = encoding
if layer:
open_options["layer"] = layer
def transformer(crs, feat):
tg = partial(
transform_geom,
crs,
"EPSG:4326",
antimeridian_cutting=True,
precision=precision,
)
return Feature(
id=feat.id, properties=feat.properties, geometry=tg(feat.geometry)
)
with fiona.open(input, **open_options) as source:
meta = source.meta
meta["fields"] = dict(source.schema["properties"].items())
if record_buffered:
# Buffer GeoJSON data at the feature level for smaller
# memory footprint.
indented = bool(indent)
rec_indent = "\n" + " " * (2 * (indent or 0))
collection = {
"type": "FeatureCollection",
"fiona:schema": meta["schema"],
"fiona:crs": meta["crs"],
"features": [],
}
if with_ld_context:
collection["@context"] = helpers.make_ld_context(add_ld_context_item)
head, tail = json.dumps(collection, **dump_kwds).split("[]")
sink.write(head)
sink.write("[")
itr = iter(source)
# Try the first record.
try:
i, first = 0, next(itr)
first = transformer(first)
if with_ld_context:
first = helpers.id_record(first)
if indented:
sink.write(rec_indent)
sink.write(
json.dumps(first, cls=ObjectEncoder, **dump_kwds).replace(
"\n", rec_indent
)
)
except StopIteration:
pass
except Exception as exc:
# Ignoring errors is *not* the default.
if ignore_errors:
logger.error(
"failed to serialize file record %d (%s), " "continuing", i, exc
)
else:
# Log error and close up the GeoJSON, leaving it
# more or less valid no matter what happens above.
logger.critical(
"failed to serialize file record %d (%s), " "quitting", i, exc
)
sink.write("]")
sink.write(tail)
if indented:
sink.write("\n")
raise
# Because trailing commas aren't valid in JSON arrays
# we'll write the item separator before each of the
# remaining features.
for i, rec in enumerate(itr, 1):
rec = transformer(rec)
try:
if with_ld_context:
rec = helpers.id_record(rec)
if indented:
sink.write(rec_indent)
sink.write(item_sep)
sink.write(
json.dumps(rec, cls=ObjectEncoder, **dump_kwds).replace(
"\n", rec_indent
)
)
except Exception as exc:
if ignore_errors:
logger.error(
"failed to serialize file record %d (%s), "
"continuing",
i, exc)
else:
logger.critical(
"failed to serialize file record %d (%s), "
"quitting",
i, exc)
sink.write("]")
sink.write(tail)
if indented:
sink.write("\n")
raise
# Close up the GeoJSON after writing all features.
sink.write("]")
sink.write(tail)
if indented:
sink.write("\n")
else:
# Buffer GeoJSON data at the collection level. The default.
collection = {
"type": "FeatureCollection",
"fiona:schema": meta["schema"],
"fiona:crs": meta["crs"].to_string(),
}
if with_ld_context:
collection["@context"] = helpers.make_ld_context(add_ld_context_item)
collection["features"] = [
helpers.id_record(transformer(rec)) for rec in source
]
else:
collection["features"] = [
transformer(source.crs, rec) for rec in source
]
json.dump(collection, sink, cls=ObjectEncoder, **dump_kwds)

View File

@@ -0,0 +1,38 @@
"""$ fio env"""
import json
import os
import click
import fiona
from fiona._env import GDALDataFinder, PROJDataFinder
@click.command(short_help="Print information about the fio environment.")
@click.option('--formats', 'key', flag_value='formats', default=True,
help="Enumerate the available formats.")
@click.option('--credentials', 'key', flag_value='credentials', default=False,
help="Print credentials.")
@click.option('--gdal-data', 'key', flag_value='gdal_data', default=False,
help="Print GDAL data path.")
@click.option('--proj-data', 'key', flag_value='proj_data', default=False,
help="Print PROJ data path.")
@click.pass_context
def env(ctx, key):
"""Print information about the Fiona environment: available
formats, etc.
"""
stdout = click.get_text_stream('stdout')
with ctx.obj['env'] as env:
if key == 'formats':
for k, v in sorted(fiona.supported_drivers.items()):
modes = ', '.join("'" + m + "'" for m in v)
stdout.write(f"{k} (modes {modes})\n")
stdout.write('\n')
elif key == 'credentials':
click.echo(json.dumps(env.session.credentials))
elif key == 'gdal_data':
click.echo(os.environ.get('GDAL_DATA') or GDALDataFinder().search())
elif key == 'proj_data':
click.echo(os.environ.get('PROJ_DATA', os.environ.get('PROJ_LIB')) or PROJDataFinder().search())

View File

@@ -0,0 +1,267 @@
"""Fiona CLI commands."""
from collections import defaultdict
from copy import copy
import itertools
import json
import logging
import warnings
import click
from cligj import use_rs_opt # type: ignore
from fiona.features import map_feature, reduce_features
from fiona.fio import with_context_env
from fiona.fio.helpers import obj_gen, eval_feature_expression # type: ignore
log = logging.getLogger(__name__)
@click.command(
"map",
short_help="Map a pipeline expression over GeoJSON features.",
)
@click.argument("pipeline")
@click.option(
"--raw",
"-r",
is_flag=True,
default=False,
help="Print raw result, do not wrap in a GeoJSON Feature.",
)
@click.option(
"--no-input",
"-n",
is_flag=True,
default=False,
help="Do not read input from stream.",
)
@click.option(
"--dump-parts",
is_flag=True,
default=False,
help="Dump parts of geometries to create new inputs before evaluating pipeline.",
)
@use_rs_opt
def map_cmd(pipeline, raw, no_input, dump_parts, use_rs):
"""Map a pipeline expression over GeoJSON features.
Given a sequence of GeoJSON features (RS-delimited or not) on stdin
this prints copies with geometries that are transformed using a
provided transformation pipeline. In "raw" output mode, this
command prints pipeline results without wrapping them in a feature
object.
The pipeline is a string that, when evaluated by fio-map, produces
a new geometry object. The pipeline consists of expressions in the
form of parenthesized lists that may contain other expressions.
The first item in a list is the name of a function or method, or an
expression that evaluates to a function. The second item is the
function's first argument or the object to which the method is
bound. The remaining list items are the positional and keyword
arguments for the named function or method. The names of the input
feature and its geometry in the context of these expressions are
"f" and "g".
For example, this pipeline expression
'(simplify (buffer g 100.0) 5.0)'
buffers input geometries and then simplifies them so that no
vertices are closer than 5 units. Keyword arguments for the shapely
methods are supported. A keyword argument is preceded by ':' and
followed immediately by its value. For example:
'(simplify g 5.0 :preserve_topology true)'
and
'(buffer g 100.0 :resolution 8 :join_style 1)'
Numerical and string arguments may be replaced by expressions. The
buffer distance could be a function of a geometry's area.
'(buffer g (/ (area g) 100.0))'
"""
if no_input:
features = [None]
else:
stdin = click.get_text_stream("stdin")
features = obj_gen(stdin)
for feat in features:
for i, value in enumerate(map_feature(pipeline, feat, dump_parts=dump_parts)):
if use_rs:
click.echo("\x1e", nl=False)
if raw:
click.echo(json.dumps(value))
else:
new_feat = copy(feat)
new_feat["id"] = f"{feat.get('id', '0')}:{i}"
new_feat["geometry"] = value
click.echo(json.dumps(new_feat))
@click.command(
"filter",
short_help="Evaluate pipeline expressions to filter GeoJSON features.",
)
@click.argument("pipeline")
@use_rs_opt
@click.option(
"--snuggs-only",
"-s",
is_flag=True,
default=False,
help="Strictly require snuggs style expressions and skip check for type of expression.",
)
@click.pass_context
@with_context_env
def filter_cmd(ctx, pipeline, use_rs, snuggs_only):
"""Evaluate pipeline expressions to filter GeoJSON features.
The pipeline is a string that, when evaluated, gives a new value for
each input feature. If the value evaluates to True, the feature
passes through the filter. Otherwise the feature does not pass.
The pipeline consists of expressions in the form of parenthesized
lists that may contain other expressions. The first item in a list
is the name of a function or method, or an expression that evaluates
to a function. The second item is the function's first argument or
the object to which the method is bound. The remaining list items
are the positional and keyword arguments for the named function or
method. The names of the input feature and its geometry in the
context of these expressions are "f" and "g".
For example, this pipeline expression
'(< (distance g (Point 4 43)) 1)'
lets through all features that are less than one unit from the given
point and filters out all other features.
*New in version 1.10*: these parenthesized list expressions.
The older style Python expressions like
'f.properties.area > 1000.0'
are deprecated and will not be supported in version 2.0.
"""
stdin = click.get_text_stream("stdin")
features = obj_gen(stdin)
if not snuggs_only:
try:
from pyparsing.exceptions import ParseException
from fiona._vendor.snuggs import ExpressionError, expr
if not pipeline.startswith("("):
test_string = f"({pipeline})"
expr.parseString(test_string)
except ExpressionError:
# It's a snuggs expression.
log.info("Detected a snuggs expression.")
pass
except ParseException:
# It's likely an old-style Python expression.
log.info("Detected a legacy Python expression.")
warnings.warn(
"This style of filter expression is deprecated. "
"Version 2.0 will only support the new parenthesized list expressions.",
FutureWarning,
)
for i, obj in enumerate(features):
feats = obj.get("features") or [obj]
for j, feat in enumerate(feats):
if not eval_feature_expression(feat, pipeline):
continue
if use_rs:
click.echo("\x1e", nl=False)
click.echo(json.dumps(feat))
return
for feat in features:
for value in map_feature(pipeline, feat):
if value:
if use_rs:
click.echo("\x1e", nl=False)
click.echo(json.dumps(feat))
@click.command("reduce", short_help="Reduce a stream of GeoJSON features to one value.")
@click.argument("pipeline")
@click.option(
"--raw",
"-r",
is_flag=True,
default=False,
help="Print raw result, do not wrap in a GeoJSON Feature.",
)
@use_rs_opt
@click.option(
"--zip-properties",
is_flag=True,
default=False,
help="Zip the items of input feature properties together for output.",
)
def reduce_cmd(pipeline, raw, use_rs, zip_properties):
"""Reduce a stream of GeoJSON features to one value.
Given a sequence of GeoJSON features (RS-delimited or not) on stdin
this prints a single value using a provided transformation pipeline.
The pipeline is a string that, when evaluated, produces
a new geometry object. The pipeline consists of expressions in the
form of parenthesized lists that may contain other expressions.
The first item in a list is the name of a function or method, or an
expression that evaluates to a function. The second item is the
function's first argument or the object to which the method is
bound. The remaining list items are the positional and keyword
arguments for the named function or method. The set of geometries
of the input features in the context of these expressions is named
"c".
For example, the pipeline expression
'(unary_union c)'
dissolves the geometries of input features.
To keep the properties of input features while reducing them to a
single feature, use the --zip-properties flag. The properties of the
input features will surface in the output feature as lists
containing the input values.
"""
stdin = click.get_text_stream("stdin")
features = (feat for feat in obj_gen(stdin))
if zip_properties:
prop_features, geom_features = itertools.tee(features)
properties = defaultdict(list)
for feat in prop_features:
for key, val in feat["properties"].items():
properties[key].append(val)
else:
geom_features = features
properties = {}
for result in reduce_features(pipeline, geom_features):
if use_rs:
click.echo("\x1e", nl=False)
if raw:
click.echo(json.dumps(result))
else:
click.echo(
json.dumps(
{
"type": "Feature",
"properties": properties,
"geometry": result,
"id": "0",
}
)
)

View File

@@ -0,0 +1,134 @@
"""Helper objects needed by multiple CLI commands.
"""
from functools import partial
import json
import math
import warnings
from fiona.model import Geometry, to_dict
from fiona._vendor.munch import munchify
warnings.simplefilter("default")
def obj_gen(lines, object_hook=None):
"""Return a generator of JSON objects loaded from ``lines``."""
first_line = next(lines)
if first_line.startswith("\x1e"):
def gen():
buffer = first_line.strip("\x1e")
for line in lines:
if line.startswith("\x1e"):
if buffer:
yield json.loads(buffer, object_hook=object_hook)
buffer = line.strip("\x1e")
else:
buffer += line
else:
yield json.loads(buffer, object_hook=object_hook)
else:
def gen():
yield json.loads(first_line, object_hook=object_hook)
for line in lines:
yield json.loads(line, object_hook=object_hook)
return gen()
def nullable(val, cast):
if val is None:
return None
else:
return cast(val)
def eval_feature_expression(feature, expression):
safe_dict = {"f": munchify(to_dict(feature))}
safe_dict.update(
{
"sum": sum,
"pow": pow,
"min": min,
"max": max,
"math": math,
"bool": bool,
"int": partial(nullable, int),
"str": partial(nullable, str),
"float": partial(nullable, float),
"len": partial(nullable, len),
}
)
try:
from shapely.geometry import shape
safe_dict["shape"] = shape
except ImportError:
pass
return eval(expression, {"__builtins__": None}, safe_dict)
def make_ld_context(context_items):
"""Returns a JSON-LD Context object.
See https://json-ld.org/spec/latest/json-ld/."""
ctx = {
"@context": {
"geojson": "http://ld.geojson.org/vocab#",
"Feature": "geojson:Feature",
"FeatureCollection": "geojson:FeatureCollection",
"GeometryCollection": "geojson:GeometryCollection",
"LineString": "geojson:LineString",
"MultiLineString": "geojson:MultiLineString",
"MultiPoint": "geojson:MultiPoint",
"MultiPolygon": "geojson:MultiPolygon",
"Point": "geojson:Point",
"Polygon": "geojson:Polygon",
"bbox": {"@container": "@list", "@id": "geojson:bbox"},
"coordinates": "geojson:coordinates",
"datetime": "http://www.w3.org/2006/time#inXSDDateTime",
"description": "http://purl.org/dc/terms/description",
"features": {"@container": "@set", "@id": "geojson:features"},
"geometry": "geojson:geometry",
"id": "@id",
"properties": "geojson:properties",
"start": "http://www.w3.org/2006/time#hasBeginning",
"stop": "http://www.w3.org/2006/time#hasEnding",
"title": "http://purl.org/dc/terms/title",
"type": "@type",
"when": "geojson:when",
}
}
for item in context_items or []:
t, uri = item.split("=")
ctx[t.strip()] = uri.strip()
return ctx
def id_record(rec):
"""Converts a record's id to a blank node id and returns the record."""
rec["id"] = f"_:f{rec['id']}"
return rec
def recursive_round(obj, precision):
"""Recursively round coordinates."""
if precision < 0:
return obj
if getattr(obj, "geometries", None):
return Geometry(
geometries=[recursive_round(part, precision) for part in obj.geometries]
)
elif getattr(obj, "coordinates", None):
return Geometry(
coordinates=[recursive_round(part, precision) for part in obj.coordinates]
)
if isinstance(obj, (int, float)):
return round(obj, precision)
else:
return [recursive_round(part, precision) for part in obj]

View File

@@ -0,0 +1,78 @@
"""$ fio info"""
import logging
import json
import click
from cligj import indent_opt
import fiona
import fiona.crs
from fiona.errors import DriverError
from fiona.fio import options, with_context_env
logger = logging.getLogger(__name__)
@click.command()
# One or more files.
@click.argument('input', required=True)
@click.option('--layer', metavar="INDEX|NAME", callback=options.cb_layer,
help="Print information about a specific layer. The first "
"layer is used by default. Layers use zero-based "
"numbering when accessed by index.")
@indent_opt
# Options to pick out a single metadata item and print it as
# a string.
@click.option('--count', 'meta_member', flag_value='count',
help="Print the count of features.")
@click.option('-f', '--format', '--driver', 'meta_member', flag_value='driver',
help="Print the format driver.")
@click.option('--crs', 'meta_member', flag_value='crs',
help="Print the CRS as a PROJ.4 string.")
@click.option('--bounds', 'meta_member', flag_value='bounds',
help="Print the boundary coordinates "
"(left, bottom, right, top).")
@click.option('--name', 'meta_member', flag_value='name',
help="Print the datasource's name.")
@options.open_opt
@click.pass_context
@with_context_env
def info(ctx, input, indent, meta_member, layer, open_options):
"""
Print information about a dataset.
When working with a multi-layer dataset the first layer is used by default.
Use the '--layer' option to select a different layer.
"""
with fiona.open(input, layer=layer, **open_options) as src:
info = src.meta
info.update(name=src.name)
try:
info.update(bounds=src.bounds)
except DriverError:
info.update(bounds=None)
logger.debug(
"Setting 'bounds' to None - driver was not able to calculate bounds"
)
try:
info.update(count=len(src))
except TypeError:
info.update(count=None)
logger.debug(
"Setting 'count' to None/null - layer does not support counting"
)
info["crs"] = src.crs.to_string()
if meta_member:
if isinstance(info[meta_member], (list, tuple)):
click.echo(" ".join(map(str, info[meta_member])))
else:
click.echo(info[meta_member])
else:
click.echo(json.dumps(info, indent=indent))

View File

@@ -0,0 +1,43 @@
"""$ fio insp"""
import code
import sys
import click
import fiona
from fiona.fio import options, with_context_env
@click.command(short_help="Open a dataset and start an interpreter.")
@click.argument("src_path", required=True)
@click.option(
"--ipython", "interpreter", flag_value="ipython", help="Use IPython as interpreter."
)
@options.open_opt
@click.pass_context
@with_context_env
def insp(ctx, src_path, interpreter, open_options):
"""Open a collection within an interactive interpreter."""
banner = (
"Fiona %s Interactive Inspector (Python %s)\n"
'Type "src.schema", "next(src)", or "help(src)" '
"for more information."
% (fiona.__version__, ".".join(map(str, sys.version_info[:3])))
)
with fiona.open(src_path, **open_options) as src:
scope = locals()
if not interpreter:
code.interact(banner, local=scope)
elif interpreter == "ipython":
import IPython
IPython.InteractiveShell.banner1 = banner
IPython.start_ipython(argv=[], user_ns=scope)
else:
raise click.ClickException(
f"Interpreter {interpreter} is unsupported or missing "
"dependencies"
)

View File

@@ -0,0 +1,114 @@
"""$ fio load"""
from functools import partial
import click
import cligj
import fiona
from fiona.fio import options, with_context_env
from fiona.model import Feature, Geometry
from fiona.transform import transform_geom
@click.command(short_help="Load GeoJSON to a dataset in another format.")
@click.argument("output", required=True)
@click.option("-f", "--format", "--driver", "driver", help="Output format driver name.")
@options.src_crs_opt
@click.option(
"--dst-crs",
"--dst_crs",
help="Destination CRS. Defaults to --src-crs when not given.",
)
@cligj.features_in_arg
@click.option(
"--layer",
metavar="INDEX|NAME",
callback=options.cb_layer,
help="Load features into specified layer. Layers use "
"zero-based numbering when accessed by index.",
)
@options.creation_opt
@options.open_opt
@click.option("--append", is_flag=True, help="Open destination layer in append mode.")
@click.pass_context
@with_context_env
def load(
ctx,
output,
driver,
src_crs,
dst_crs,
features,
layer,
creation_options,
open_options,
append,
):
"""Load features from JSON to a file in another format.
The input is a GeoJSON feature collection or optionally a sequence of
GeoJSON feature objects.
"""
dst_crs = dst_crs or src_crs
if src_crs and dst_crs and src_crs != dst_crs:
transformer = partial(
transform_geom, src_crs, dst_crs, antimeridian_cutting=True
)
else:
def transformer(x):
return Geometry.from_dict(**x)
def feature_gen():
"""Convert stream of JSON to features.
Yields
------
Feature
"""
try:
for feat in features:
feat["geometry"] = transformer(Geometry.from_dict(**feat["geometry"]))
yield Feature.from_dict(**feat)
except TypeError:
raise click.ClickException("Invalid input.")
source = feature_gen()
# Use schema of first feature as a template.
# TODO: schema specified on command line?
try:
first = next(source)
except TypeError:
raise click.ClickException("Invalid input.")
# TODO: this inference of a property's type from its value needs some
# work. It works reliably only for the basic JSON serializable types.
# The fio-load command does require JSON input but that may change
# someday.
schema = {"geometry": first.geometry.type}
schema["properties"] = {
k: type(v if v is not None else "").__name__
for k, v in first.properties.items()
}
if append:
opener = fiona.open(output, "a", layer=layer, **open_options)
else:
opener = fiona.open(
output,
"w",
driver=driver,
crs=dst_crs,
schema=schema,
layer=layer,
**creation_options
)
with opener as dst:
dst.write(first)
dst.writerecords(source)

View File

@@ -0,0 +1,24 @@
"""$ fiona ls"""
import json
import click
from cligj import indent_opt
import fiona
from fiona.fio import options, with_context_env
@click.command()
@click.argument('input', required=True)
@indent_opt
@options.open_opt
@click.pass_context
@with_context_env
def ls(ctx, input, indent, open_options):
"""
List layers in a datasource.
"""
result = fiona.listlayers(input, **open_options)
click.echo(json.dumps(result, indent=indent))

View File

@@ -0,0 +1,113 @@
"""
Main click group for the CLI. Needs to be isolated for entry-point loading.
"""
import itertools
import logging
import sys
import click
from click_plugins import with_plugins
from cligj import verbose_opt, quiet_opt
if sys.version_info < (3, 10):
from importlib_metadata import entry_points
else:
from importlib.metadata import entry_points
import fiona
from fiona import __version__ as fio_version
from fiona.session import AWSSession, DummySession
from fiona.fio.bounds import bounds
from fiona.fio.calc import calc
from fiona.fio.cat import cat
from fiona.fio.collect import collect
from fiona.fio.distrib import distrib
from fiona.fio.dump import dump
from fiona.fio.env import env
from fiona.fio.info import info
from fiona.fio.insp import insp
from fiona.fio.load import load
from fiona.fio.ls import ls
from fiona.fio.rm import rm
# The "calc" extras require pyparsing and shapely.
try:
import pyparsing
import shapely
from fiona.fio.features import filter_cmd, map_cmd, reduce_cmd
supports_calc = True
except ImportError:
supports_calc = False
def configure_logging(verbosity):
log_level = max(10, 30 - 10 * verbosity)
logging.basicConfig(stream=sys.stderr, level=log_level)
@with_plugins(
itertools.chain(
entry_points(group="fiona.fio_plugins"),
)
)
@click.group()
@verbose_opt
@quiet_opt
@click.option(
"--aws-profile",
help="Select a profile from the AWS credentials file")
@click.option(
"--aws-no-sign-requests",
is_flag=True,
help="Make requests anonymously")
@click.option(
"--aws-requester-pays",
is_flag=True,
help="Requester pays data transfer costs")
@click.version_option(fio_version)
@click.version_option(fiona.__gdal_version__, '--gdal-version',
prog_name='GDAL')
@click.version_option(sys.version, '--python-version', prog_name='Python')
@click.pass_context
def main_group(
ctx, verbose, quiet, aws_profile, aws_no_sign_requests,
aws_requester_pays):
"""Fiona command line interface.
"""
verbosity = verbose - quiet
configure_logging(verbosity)
ctx.obj = {}
ctx.obj["verbosity"] = verbosity
ctx.obj["aws_profile"] = aws_profile
envopts = {"CPL_DEBUG": (verbosity > 2)}
if aws_profile or aws_no_sign_requests:
session = AWSSession(
profile_name=aws_profile,
aws_unsigned=aws_no_sign_requests,
requester_pays=aws_requester_pays,
)
else:
session = DummySession()
ctx.obj["env"] = fiona.Env(session=session, **envopts)
main_group.add_command(bounds)
main_group.add_command(calc)
main_group.add_command(cat)
main_group.add_command(collect)
main_group.add_command(distrib)
main_group.add_command(dump)
main_group.add_command(env)
main_group.add_command(info)
main_group.add_command(insp)
main_group.add_command(load)
main_group.add_command(ls)
main_group.add_command(rm)
if supports_calc:
main_group.add_command(map_cmd)
main_group.add_command(filter_cmd)
main_group.add_command(reduce_cmd)

View File

@@ -0,0 +1,96 @@
"""Common commandline options for `fio`"""
from collections import defaultdict
import click
src_crs_opt = click.option('--src-crs', '--src_crs', help="Source CRS.")
dst_crs_opt = click.option('--dst-crs', '--dst_crs', help="Destination CRS.")
def cb_layer(ctx, param, value):
"""Let --layer be a name or index."""
if value is None or not value.isdigit():
return value
else:
return int(value)
def cb_multilayer(ctx, param, value):
"""
Transform layer options from strings ("1:a,1:b", "2:a,2:c,2:z") to
{
'1': ['a', 'b'],
'2': ['a', 'c', 'z']
}
"""
out = defaultdict(list)
for raw in value:
for v in raw.split(','):
ds, name = v.split(':')
out[ds].append(name)
return out
def cb_key_val(ctx, param, value):
"""
click callback to validate `--opt KEY1=VAL1 --opt KEY2=VAL2` and collect
in a dictionary like the one below, which is what the CLI function receives.
If no value or `None` is received then an empty dictionary is returned.
{
'KEY1': 'VAL1',
'KEY2': 'VAL2'
}
Note: `==VAL` breaks this as `str.split('=', 1)` is used.
"""
if not value:
return {}
else:
out = {}
for pair in value:
if "=" not in pair:
raise click.BadParameter(
f"Invalid syntax for KEY=VAL arg: {pair}"
)
else:
k, v = pair.split("=", 1)
k = k.lower()
v = v.lower()
out[k] = None if v.lower() in ["none", "null", "nil", "nada"] else v
return out
def validate_multilayer_file_index(files, layerdict):
"""
Ensure file indexes provided in the --layer option are valid
"""
for key in layerdict.keys():
if key not in [str(k) for k in range(1, len(files) + 1)]:
layer = key + ":" + layerdict[key][0]
raise click.BadParameter(f"Layer {layer} does not exist")
creation_opt = click.option(
"--co",
"--profile",
"creation_options",
metavar="NAME=VALUE",
multiple=True,
callback=cb_key_val,
help="Driver specific creation options. See the documentation for the selected output driver for more information.",
)
open_opt = click.option(
"--oo",
"open_options",
metavar="NAME=VALUE",
multiple=True,
callback=cb_key_val,
help="Driver specific open options. See the documentation for the selected output driver for more information.",
)

View File

@@ -0,0 +1,30 @@
import click
import logging
import fiona
from fiona.fio import with_context_env
logger = logging.getLogger(__name__)
@click.command(help="Remove a datasource or an individual layer.")
@click.argument("input", required=True)
@click.option("--layer", type=str, default=None, required=False, help="Name of layer to remove.")
@click.option("--yes", is_flag=True)
@click.pass_context
@with_context_env
def rm(ctx, input, layer, yes):
if layer is None:
kind = "datasource"
else:
kind = "layer"
if not yes:
click.confirm(f"The {kind} will be removed. Are you sure?", abort=True)
try:
fiona.remove(input, layer=layer)
except Exception:
logger.exception(f"Failed to remove {kind}.")
raise click.Abort()