Run downsampling with ILM
This is a simplified example that allows you to see quickly how downsampling works as part of an ILM policy to reduce the storage size of a sampled set of metrics. The example uses typical Kubernetes cluster monitoring data. To test out downsampling with ILM, follow these steps:
Prerequisites
Refer to time series data stream prerequisites.
Before running this example you may want to try the Run downsampling manually example.
Create an index lifecycle policy
Create an ILM policy for your time series data. While not required, an ILM policy is recommended to automate the management of your time series data stream indices.
To enable downsampling, add a Downsample action and set fixed_interval to the downsampling interval at which you want to aggregate the original time series data.
In this example, an ILM policy is configured for the hot phase. The downsample takes place after the index is rolled over and the index time series end time has lapsed as the source index is still expected to receive major writes until then. Index lifecycle management will not proceed with any action that expects the index to not receive writes anymore until the index’s end time has passed. The Index lifecycle management actions that wait on the end time before proceeding are: - Delete - Downsample - Force merge - Read only - Searchable snapshot - Shrink
resp = client.ilm.put_lifecycle(
name="datastream_policy",
policy={
"phases": {
"hot": {
"actions": {
"rollover": {
"max_age": "5m"
},
"downsample": {
"fixed_interval": "1h"
}
}
}
}
},
)
print(resp) response = client.ilm.put_lifecycle(
policy: 'datastream_policy',
body: {
policy: {
phases: {
hot: {
actions: {
rollover: {
max_age: '5m'
},
downsample: {
fixed_interval: '1h'
}
}
}
}
}
}
)
puts response const response = await client.ilm.putLifecycle({
name: "datastream_policy",
policy: {
phases: {
hot: {
actions: {
rollover: {
max_age: "5m",
},
downsample: {
fixed_interval: "1h",
},
},
},
},
},
});
console.log(response); PUT _ilm/policy/datastream_policy
{
"policy": {
"phases": {
"hot": {
"actions": {
"rollover" : {
"max_age": "5m"
},
"downsample": {
"fixed_interval": "1h"
}
}
}
}
}
} Create an index template
This creates an index template for a basic data stream. The available parameters for an index template are described in detail in Set up a time series data stream.
For simplicity, in the time series mapping all time_series_metric parameters are set to type gauge, but the counter metric type may also be used. The time_series_metric values determine the kind of statistical representations that are used during downsampling.
The index template includes a set of static time series dimensions: host, namespace, node, and pod. The time series dimensions are not changed by the downsampling process.
resp = client.indices.put_index_template(
name="datastream_template",
index_patterns=[
"datastream*"
],
data_stream={},
template={
"settings": {
"index": {
"mode": "time_series",
"number_of_replicas": 0,
"number_of_shards": 2
},
"index.lifecycle.name": "datastream_policy"
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"kubernetes": {
"properties": {
"container": {
"properties": {
"cpu": {
"properties": {
"usage": {
"properties": {
"core": {
"properties": {
"ns": {
"type": "long"
}
}
},
"limit": {
"properties": {
"pct": {
"type": "float"
}
}
},
"nanocores": {
"type": "long",
"time_series_metric": "gauge"
},
"node": {
"properties": {
"pct": {
"type": "float"
}
}
}
}
}
}
},
"memory": {
"properties": {
"available": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
},
"majorpagefaults": {
"type": "long"
},
"pagefaults": {
"type": "long",
"time_series_metric": "gauge"
},
"rss": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
},
"usage": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
},
"limit": {
"properties": {
"pct": {
"type": "float"
}
}
},
"node": {
"properties": {
"pct": {
"type": "float"
}
}
}
}
},
"workingset": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
}
}
},
"name": {
"type": "keyword"
},
"start_time": {
"type": "date"
}
}
},
"host": {
"type": "keyword",
"time_series_dimension": True
},
"namespace": {
"type": "keyword",
"time_series_dimension": True
},
"node": {
"type": "keyword",
"time_series_dimension": True
},
"pod": {
"type": "keyword",
"time_series_dimension": True
}
}
}
}
}
},
)
print(resp) response = client.indices.put_index_template(
name: 'datastream_template',
body: {
index_patterns: [
'datastream*'
],
data_stream: {},
template: {
settings: {
index: {
mode: 'time_series',
number_of_replicas: 0,
number_of_shards: 2
},
'index.lifecycle.name' => 'datastream_policy'
},
mappings: {
properties: {
"@timestamp": {
type: 'date'
},
kubernetes: {
properties: {
container: {
properties: {
cpu: {
properties: {
usage: {
properties: {
core: {
properties: {
ns: {
type: 'long'
}
}
},
limit: {
properties: {
pct: {
type: 'float'
}
}
},
nanocores: {
type: 'long',
time_series_metric: 'gauge'
},
node: {
properties: {
pct: {
type: 'float'
}
}
}
}
}
}
},
memory: {
properties: {
available: {
properties: {
bytes: {
type: 'long',
time_series_metric: 'gauge'
}
}
},
majorpagefaults: {
type: 'long'
},
pagefaults: {
type: 'long',
time_series_metric: 'gauge'
},
rss: {
properties: {
bytes: {
type: 'long',
time_series_metric: 'gauge'
}
}
},
usage: {
properties: {
bytes: {
type: 'long',
time_series_metric: 'gauge'
},
limit: {
properties: {
pct: {
type: 'float'
}
}
},
node: {
properties: {
pct: {
type: 'float'
}
}
}
}
},
workingset: {
properties: {
bytes: {
type: 'long',
time_series_metric: 'gauge'
}
}
}
}
},
name: {
type: 'keyword'
},
start_time: {
type: 'date'
}
}
},
host: {
type: 'keyword',
time_series_dimension: true
},
namespace: {
type: 'keyword',
time_series_dimension: true
},
node: {
type: 'keyword',
time_series_dimension: true
},
pod: {
type: 'keyword',
time_series_dimension: true
}
}
}
}
}
}
}
)
puts response const response = await client.indices.putIndexTemplate({
name: "datastream_template",
index_patterns: ["datastream*"],
data_stream: {},
template: {
settings: {
index: {
mode: "time_series",
number_of_replicas: 0,
number_of_shards: 2,
},
"index.lifecycle.name": "datastream_policy",
},
mappings: {
properties: {
"@timestamp": {
type: "date",
},
kubernetes: {
properties: {
container: {
properties: {
cpu: {
properties: {
usage: {
properties: {
core: {
properties: {
ns: {
type: "long",
},
},
},
limit: {
properties: {
pct: {
type: "float",
},
},
},
nanocores: {
type: "long",
time_series_metric: "gauge",
},
node: {
properties: {
pct: {
type: "float",
},
},
},
},
},
},
},
memory: {
properties: {
available: {
properties: {
bytes: {
type: "long",
time_series_metric: "gauge",
},
},
},
majorpagefaults: {
type: "long",
},
pagefaults: {
type: "long",
time_series_metric: "gauge",
},
rss: {
properties: {
bytes: {
type: "long",
time_series_metric: "gauge",
},
},
},
usage: {
properties: {
bytes: {
type: "long",
time_series_metric: "gauge",
},
limit: {
properties: {
pct: {
type: "float",
},
},
},
node: {
properties: {
pct: {
type: "float",
},
},
},
},
},
workingset: {
properties: {
bytes: {
type: "long",
time_series_metric: "gauge",
},
},
},
},
},
name: {
type: "keyword",
},
start_time: {
type: "date",
},
},
},
host: {
type: "keyword",
time_series_dimension: true,
},
namespace: {
type: "keyword",
time_series_dimension: true,
},
node: {
type: "keyword",
time_series_dimension: true,
},
pod: {
type: "keyword",
time_series_dimension: true,
},
},
},
},
},
},
});
console.log(response); PUT _index_template/datastream_template
{
"index_patterns": [
"datastream*"
],
"data_stream": {},
"template": {
"settings": {
"index": {
"mode": "time_series",
"number_of_replicas": 0,
"number_of_shards": 2
},
"index.lifecycle.name": "datastream_policy"
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"kubernetes": {
"properties": {
"container": {
"properties": {
"cpu": {
"properties": {
"usage": {
"properties": {
"core": {
"properties": {
"ns": {
"type": "long"
}
}
},
"limit": {
"properties": {
"pct": {
"type": "float"
}
}
},
"nanocores": {
"type": "long",
"time_series_metric": "gauge"
},
"node": {
"properties": {
"pct": {
"type": "float"
}
}
}
}
}
}
},
"memory": {
"properties": {
"available": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
},
"majorpagefaults": {
"type": "long"
},
"pagefaults": {
"type": "long",
"time_series_metric": "gauge"
},
"rss": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
},
"usage": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
},
"limit": {
"properties": {
"pct": {
"type": "float"
}
}
},
"node": {
"properties": {
"pct": {
"type": "float"
}
}
}
}
},
"workingset": {
"properties": {
"bytes": {
"type": "long",
"time_series_metric": "gauge"
}
}
}
}
},
"name": {
"type": "keyword"
},
"start_time": {
"type": "date"
}
}
},
"host": {
"type": "keyword",
"time_series_dimension": true
},
"namespace": {
"type": "keyword",
"time_series_dimension": true
},
"node": {
"type": "keyword",
"time_series_dimension": true
},
"pod": {
"type": "keyword",
"time_series_dimension": true
}
}
}
}
}
}
} Ingest time series data
Use a bulk API request to automatically create your TSDS and index a set of ten documents.
Important: Before running this bulk request you need to update the timestamps to within three to five hours after your current time. That is, search 2022-06-21T15 and replace with your present date, and adjust the hour to your current time plus three hours.
resp = client.bulk(
index="datastream",
refresh=True,
operations=[
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:49:00Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 91153,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 463314616
},
"usage": {
"bytes": 307007078,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 585236
},
"rss": {
"bytes": 102728
},
"pagefaults": 120901,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:45:50Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 124501,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 982546514
},
"usage": {
"bytes": 360035574,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 1339884
},
"rss": {
"bytes": 381174
},
"pagefaults": 178473,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:44:50Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 38907,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 862723768
},
"usage": {
"bytes": 379572388,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 431227
},
"rss": {
"bytes": 386580
},
"pagefaults": 233166,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:44:40Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 86706,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 567160996
},
"usage": {
"bytes": 103266017,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 1724908
},
"rss": {
"bytes": 105431
},
"pagefaults": 233166,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:44:00Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 150069,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 639054643
},
"usage": {
"bytes": 265142477,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 1786511
},
"rss": {
"bytes": 189235
},
"pagefaults": 138172,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:42:40Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 82260,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 854735585
},
"usage": {
"bytes": 309798052,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 924058
},
"rss": {
"bytes": 110838
},
"pagefaults": 259073,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:42:10Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 153404,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 279586406
},
"usage": {
"bytes": 214904955,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 1047265
},
"rss": {
"bytes": 91914
},
"pagefaults": 302252,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:40:20Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 125613,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 822782853
},
"usage": {
"bytes": 100475044,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 2109932
},
"rss": {
"bytes": 278446
},
"pagefaults": 74843,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:40:10Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 100046,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 567160996
},
"usage": {
"bytes": 362826547,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 1986724
},
"rss": {
"bytes": 402801
},
"pagefaults": 296495,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
},
{
"create": {}
},
{
"@timestamp": "2022-06-21T15:38:30Z",
"kubernetes": {
"host": "gke-apps-0",
"node": "gke-apps-0-0",
"pod": "gke-apps-0-0-0",
"container": {
"cpu": {
"usage": {
"nanocores": 40018,
"core": {
"ns": 12828317850
},
"node": {
"pct": 0.0000277905
},
"limit": {
"pct": 0.0000277905
}
}
},
"memory": {
"available": {
"bytes": 1062428344
},
"usage": {
"bytes": 265142477,
"node": {
"pct": 0.01770037710617187
},
"limit": {
"pct": 0.00009923134671484496
}
},
"workingset": {
"bytes": 2294743
},
"rss": {
"bytes": 340623
},
"pagefaults": 224530,
"majorpagefaults": 0
},
"start_time": "2021-03-30T07:59:06Z",
"name": "container-name-44"
},
"namespace": "namespace26"
}
}
],
)
print(resp) response = client.bulk(
index: 'datastream',
refresh: true,
body: [
{
create: {}
},
{
"@timestamp": '2022-06-21T15:49:00Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 91_153,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 463_314_616
},
usage: {
bytes: 307_007_078,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 585_236
},
rss: {
bytes: 102_728
},
pagefaults: 120_901,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:45:50Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 124_501,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 982_546_514
},
usage: {
bytes: 360_035_574,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_339_884
},
rss: {
bytes: 381_174
},
pagefaults: 178_473,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:44:50Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 38_907,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 862_723_768
},
usage: {
bytes: 379_572_388,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 431_227
},
rss: {
bytes: 386_580
},
pagefaults: 233_166,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:44:40Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 86_706,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 567_160_996
},
usage: {
bytes: 103_266_017,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_724_908
},
rss: {
bytes: 105_431
},
pagefaults: 233_166,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:44:00Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 150_069,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 639_054_643
},
usage: {
bytes: 265_142_477,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_786_511
},
rss: {
bytes: 189_235
},
pagefaults: 138_172,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:42:40Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 82_260,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 854_735_585
},
usage: {
bytes: 309_798_052,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 924_058
},
rss: {
bytes: 110_838
},
pagefaults: 259_073,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:42:10Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 153_404,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 279_586_406
},
usage: {
bytes: 214_904_955,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_047_265
},
rss: {
bytes: 91_914
},
pagefaults: 302_252,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:40:20Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 125_613,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 822_782_853
},
usage: {
bytes: 100_475_044,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 2_109_932
},
rss: {
bytes: 278_446
},
pagefaults: 74_843,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:40:10Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 100_046,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 567_160_996
},
usage: {
bytes: 362_826_547,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 1_986_724
},
rss: {
bytes: 402_801
},
pagefaults: 296_495,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
},
{
create: {}
},
{
"@timestamp": '2022-06-21T15:38:30Z',
kubernetes: {
host: 'gke-apps-0',
node: 'gke-apps-0-0',
pod: 'gke-apps-0-0-0',
container: {
cpu: {
usage: {
nanocores: 40_018,
core: {
ns: 12_828_317_850
},
node: {
pct: 2.77905e-05
},
limit: {
pct: 2.77905e-05
}
}
},
memory: {
available: {
bytes: 1_062_428_344
},
usage: {
bytes: 265_142_477,
node: {
pct: 0.01770037710617187
},
limit: {
pct: 9.923134671484496e-05
}
},
workingset: {
bytes: 2_294_743
},
rss: {
bytes: 340_623
},
pagefaults: 224_530,
majorpagefaults: 0
},
start_time: '2021-03-30T07:59:06Z',
name: 'container-name-44'
},
namespace: 'namespace26'
}
}
]
)
puts response const response = await client.bulk({
index: "datastream",
refresh: "true",
operations: [
{
create: {},
},
{
"@timestamp": "2022-06-21T15:49:00Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 91153,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 463314616,
},
usage: {
bytes: 307007078,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 585236,
},
rss: {
bytes: 102728,
},
pagefaults: 120901,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:45:50Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 124501,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 982546514,
},
usage: {
bytes: 360035574,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 1339884,
},
rss: {
bytes: 381174,
},
pagefaults: 178473,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:44:50Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 38907,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 862723768,
},
usage: {
bytes: 379572388,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 431227,
},
rss: {
bytes: 386580,
},
pagefaults: 233166,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:44:40Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 86706,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 567160996,
},
usage: {
bytes: 103266017,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 1724908,
},
rss: {
bytes: 105431,
},
pagefaults: 233166,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:44:00Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 150069,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 639054643,
},
usage: {
bytes: 265142477,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 1786511,
},
rss: {
bytes: 189235,
},
pagefaults: 138172,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:42:40Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 82260,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 854735585,
},
usage: {
bytes: 309798052,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 924058,
},
rss: {
bytes: 110838,
},
pagefaults: 259073,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:42:10Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 153404,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 279586406,
},
usage: {
bytes: 214904955,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 1047265,
},
rss: {
bytes: 91914,
},
pagefaults: 302252,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:40:20Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 125613,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 822782853,
},
usage: {
bytes: 100475044,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 2109932,
},
rss: {
bytes: 278446,
},
pagefaults: 74843,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:40:10Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 100046,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 567160996,
},
usage: {
bytes: 362826547,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 1986724,
},
rss: {
bytes: 402801,
},
pagefaults: 296495,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
{
create: {},
},
{
"@timestamp": "2022-06-21T15:38:30Z",
kubernetes: {
host: "gke-apps-0",
node: "gke-apps-0-0",
pod: "gke-apps-0-0-0",
container: {
cpu: {
usage: {
nanocores: 40018,
core: {
ns: 12828317850,
},
node: {
pct: 0.0000277905,
},
limit: {
pct: 0.0000277905,
},
},
},
memory: {
available: {
bytes: 1062428344,
},
usage: {
bytes: 265142477,
node: {
pct: 0.01770037710617187,
},
limit: {
pct: 0.00009923134671484496,
},
},
workingset: {
bytes: 2294743,
},
rss: {
bytes: 340623,
},
pagefaults: 224530,
majorpagefaults: 0,
},
start_time: "2021-03-30T07:59:06Z",
name: "container-name-44",
},
namespace: "namespace26",
},
},
],
});
console.log(response); PUT /datastream/_bulk?refresh
{"create": {}}
{"@timestamp":"2022-06-21T15:49:00Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":91153,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":463314616},"usage":{"bytes":307007078,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":585236},"rss":{"bytes":102728},"pagefaults":120901,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:45:50Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":124501,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":982546514},"usage":{"bytes":360035574,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1339884},"rss":{"bytes":381174},"pagefaults":178473,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:44:50Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":38907,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":862723768},"usage":{"bytes":379572388,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":431227},"rss":{"bytes":386580},"pagefaults":233166,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:44:40Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":86706,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":567160996},"usage":{"bytes":103266017,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1724908},"rss":{"bytes":105431},"pagefaults":233166,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:44:00Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":150069,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":639054643},"usage":{"bytes":265142477,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1786511},"rss":{"bytes":189235},"pagefaults":138172,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:42:40Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":82260,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":854735585},"usage":{"bytes":309798052,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":924058},"rss":{"bytes":110838},"pagefaults":259073,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:42:10Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":153404,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":279586406},"usage":{"bytes":214904955,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1047265},"rss":{"bytes":91914},"pagefaults":302252,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:40:20Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":125613,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":822782853},"usage":{"bytes":100475044,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":2109932},"rss":{"bytes":278446},"pagefaults":74843,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:40:10Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":100046,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":567160996},"usage":{"bytes":362826547,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":1986724},"rss":{"bytes":402801},"pagefaults":296495,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}}
{"create": {}}
{"@timestamp":"2022-06-21T15:38:30Z","kubernetes":{"host":"gke-apps-0","node":"gke-apps-0-0","pod":"gke-apps-0-0-0","container":{"cpu":{"usage":{"nanocores":40018,"core":{"ns":12828317850},"node":{"pct":2.77905e-05},"limit":{"pct":2.77905e-05}}},"memory":{"available":{"bytes":1062428344},"usage":{"bytes":265142477,"node":{"pct":0.01770037710617187},"limit":{"pct":9.923134671484496e-05}},"workingset":{"bytes":2294743},"rss":{"bytes":340623},"pagefaults":224530,"majorpagefaults":0},"start_time":"2021-03-30T07:59:06Z","name":"container-name-44"},"namespace":"namespace26"}} View the results
Now that you’ve created and added documents to the data stream, check to confirm the current state of the new index.
resp = client.indices.get_data_stream() print(resp)
response = client.indices.get_data_stream puts response
const response = await client.indices.getDataStream(); console.log(response);
GET _data_stream
If the ILM policy has not yet been applied, your results will be like the following. Note the original index_name: .ds-datastream-<timestamp>-000001.
{
"data_streams": [
{
"name": "datastream",
"timestamp_field": {
"name": "@timestamp"
},
"indices": [
{
"index_name": ".ds-datastream-2022.08.26-000001",
"index_uuid": "5g-3HrfETga-5EFKBM6R-w"
},
{
"index_name": ".ds-datastream-2022.08.26-000002",
"index_uuid": "o0yRTdhWSo2pY8XMvfwy7Q"
}
],
"generation": 2,
"status": "GREEN",
"template": "datastream_template",
"ilm_policy": "datastream_policy",
"hidden": false,
"system": false,
"allow_custom_routing": false,
"replicated": false,
"rollover_on_write": false,
"time_series": {
"temporal_ranges": [
{
"start": "2022-08-26T13:29:07.000Z",
"end": "2022-08-26T19:29:07.000Z"
}
]
}
}
]
} Next, run a search query:
resp = client.search(
index="datastream",
)
print(resp) response = client.search( index: 'datastream' ) puts response
const response = await client.search({
index: "datastream",
});
console.log(response); GET datastream/_search
The query returns your ten newly added documents.
{
"took": 17,
"timed_out": false,
"_shards": {
"total": 4,
"successful": 4,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 10,
"relation": "eq"
},
... By default, index lifecycle management checks every ten minutes for indices that meet policy criteria. Wait for about ten minutes (maybe brew up a quick coffee or tea ☕ ) and then re-run the GET _data_stream request.
resp = client.indices.get_data_stream() print(resp)
response = client.indices.get_data_stream puts response
const response = await client.indices.getDataStream(); console.log(response);
GET _data_stream
After the ILM policy has taken effect, the original .ds-datastream-2022.08.26-000001 index is replaced with a new, downsampled index, in this case downsample-6tkn-.ds-datastream-2022.08.26-000001.
{
"data_streams": [
{
"name": "datastream",
"timestamp_field": {
"name": "@timestamp"
},
"indices": [
{
"index_name": "downsample-6tkn-.ds-datastream-2022.08.26-000001",
"index_uuid": "qRane1fQQDCNgKQhXmTIvg"
},
{
"index_name": ".ds-datastream-2022.08.26-000002",
"index_uuid": "o0yRTdhWSo2pY8XMvfwy7Q"
}
],
... Run a search query on the datastream (note that when querying downsampled indices there are a few nuances to be aware of).
resp = client.search(
index="datastream",
)
print(resp) response = client.search( index: 'datastream' ) puts response
const response = await client.search({
index: "datastream",
});
console.log(response); GET datastream/_search
The new downsampled index contains just one document that includes the min, max, sum, and value_count statistics based off of the original sampled metrics.
{
"took": 6,
"timed_out": false,
"_shards": {
"total": 4,
"successful": 4,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 1,
"relation": "eq"
},
"max_score": 1,
"hits": [
{
"_index": "downsample-6tkn-.ds-datastream-2022.08.26-000001",
"_id": "0eL0wC_4-45SnTNFAAABgtpz0wA",
"_score": 1,
"_source": {
"@timestamp": "2022-08-26T14:00:00.000Z",
"_doc_count": 10,
"kubernetes.host": "gke-apps-0",
"kubernetes.namespace": "namespace26",
"kubernetes.node": "gke-apps-0-0",
"kubernetes.pod": "gke-apps-0-0-0",
"kubernetes.container.cpu.usage.nanocores": {
"min": 38907,
"max": 153404,
"sum": 992677,
"value_count": 10
},
"kubernetes.container.memory.available.bytes": {
"min": 279586406,
"max": 1062428344,
"sum": 7101494721,
"value_count": 10
},
"kubernetes.container.memory.pagefaults": {
"min": 74843,
"max": 302252,
"sum": 2061071,
"value_count": 10
},
"kubernetes.container.memory.rss.bytes": {
"min": 91914,
"max": 402801,
"sum": 2389770,
"value_count": 10
},
"kubernetes.container.memory.usage.bytes": {
"min": 100475044,
"max": 379572388,
"sum": 2668170609,
"value_count": 10
},
"kubernetes.container.memory.workingset.bytes": {
"min": 431227,
"max": 2294743,
"sum": 14230488,
"value_count": 10
},
"kubernetes.container.cpu.usage.core.ns": 12828317850,
"kubernetes.container.cpu.usage.limit.pct": 0.000027790500098490156,
"kubernetes.container.cpu.usage.node.pct": 0.000027790500098490156,
"kubernetes.container.memory.majorpagefaults": 0,
"kubernetes.container.memory.usage.limit.pct": 0.00009923134348355234,
"kubernetes.container.memory.usage.node.pct": 0.017700377851724625,
"kubernetes.container.name": "container-name-44",
"kubernetes.container.start_time": "2021-03-30T07:59:06.000Z"
}
}
]
}
} Use the data stream stats API to get statistics for the data stream, including the storage size.
resp = client.indices.data_streams_stats(
name="datastream",
human=True,
)
print(resp) response = client.indices.data_streams_stats( name: 'datastream', human: true ) puts response
const response = await client.indices.dataStreamsStats({
name: "datastream",
human: "true",
});
console.log(response); GET /_data_stream/datastream/_stats?human=true
{
"_shards": {
"total": 4,
"successful": 4,
"failed": 0
},
"data_stream_count": 1,
"backing_indices": 2,
"total_store_size": "16.6kb",
"total_store_size_bytes": 17059,
"data_streams": [
{
"data_stream": "datastream",
"backing_indices": 2,
"store_size": "16.6kb",
"store_size_bytes": 17059,
"maximum_timestamp": 1661522400000
}
]
} This example demonstrates how downsampling works as part of an ILM policy to reduce the storage size of metrics data as it becomes less current and less frequently queried.
You can also try our Run downsampling manually example to learn how downsampling can work outside of an ILM policy.
© 2023-2025 Elasticsearch
As of September 2024, Elasticsearch is available under a choice of three licenses: the Server Side Public License (SSPL), the Elastic License, or the AGPLv3 (OSI approved).
Elasticsearch and the Elasticsearch logo are trademarks of Elasticsearch B.V., registered in the U.S. and in other countries.
https://www.elastic.co/guide/en/elasticsearch/reference/8.17/downsampling-ilm.html
