1# Copyright 2020 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15apiVersion: apiextensions.k8s.io/v1
16kind: CustomResourceDefinition
17metadata:
18 annotations:
19 cnrm.cloud.google.com/version: 1.106.0
20 creationTimestamp: null
21 labels:
22 cnrm.cloud.google.com/managed-by-kcc: "true"
23 cnrm.cloud.google.com/stability-level: alpha
24 cnrm.cloud.google.com/system: "true"
25 cnrm.cloud.google.com/tf2crd: "true"
26 name: datastreamstreams.datastream.cnrm.cloud.google.com
27spec:
28 group: datastream.cnrm.cloud.google.com
29 names:
30 categories:
31 - gcp
32 kind: DatastreamStream
33 plural: datastreamstreams
34 shortNames:
35 - gcpdatastreamstream
36 - gcpdatastreamstreams
37 singular: datastreamstream
38 scope: Namespaced
39 versions:
40 - additionalPrinterColumns:
41 - jsonPath: .metadata.creationTimestamp
42 name: Age
43 type: date
44 - description: When 'True', the most recent reconcile of the resource succeeded
45 jsonPath: .status.conditions[?(@.type=='Ready')].status
46 name: Ready
47 type: string
48 - description: The reason for the value in 'Ready'
49 jsonPath: .status.conditions[?(@.type=='Ready')].reason
50 name: Status
51 type: string
52 - description: The last transition time for the value in 'Status'
53 jsonPath: .status.conditions[?(@.type=='Ready')].lastTransitionTime
54 name: Status Age
55 type: date
56 name: v1alpha1
57 schema:
58 openAPIV3Schema:
59 properties:
60 apiVersion:
61 description: 'apiVersion defines the versioned schema of this representation
62 of an object. Servers should convert recognized schemas to the latest
63 internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources'
64 type: string
65 kind:
66 description: 'kind is a string value representing the REST resource this
67 object represents. Servers may infer this from the endpoint the client
68 submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds'
69 type: string
70 metadata:
71 type: object
72 spec:
73 properties:
74 backfillAll:
75 description: Backfill strategy to automatically backfill the Stream's
76 objects. Specific objects can be excluded.
77 properties:
78 mysqlExcludedObjects:
79 description: MySQL data source objects to avoid backfilling.
80 properties:
81 mysqlDatabases:
82 description: MySQL databases on the server.
83 items:
84 properties:
85 database:
86 description: Database name.
87 type: string
88 mysqlTables:
89 description: Tables in the database.
90 items:
91 properties:
92 mysqlColumns:
93 description: MySQL columns in the schema. When
94 unspecified as part of include/exclude objects,
95 includes/excludes everything.
96 items:
97 properties:
98 collation:
99 description: Column collation.
100 type: string
101 column:
102 description: Column name.
103 type: string
104 dataType:
105 description: |-
106 The MySQL data type. Full data types list can be found here:
107 https://dev.mysql.com/doc/refman/8.0/en/data-types.html.
108 type: string
109 length:
110 description: Column length.
111 type: integer
112 nullable:
113 description: Whether or not the column can
114 accept a null value.
115 type: boolean
116 ordinalPosition:
117 description: The ordinal position of the
118 column in the table.
119 type: integer
120 primaryKey:
121 description: Whether or not the column represents
122 a primary key.
123 type: boolean
124 type: object
125 type: array
126 table:
127 description: Table name.
128 type: string
129 required:
130 - table
131 type: object
132 type: array
133 required:
134 - database
135 type: object
136 type: array
137 required:
138 - mysqlDatabases
139 type: object
140 oracleExcludedObjects:
141 description: PostgreSQL data source objects to avoid backfilling.
142 properties:
143 oracleSchemas:
144 description: Oracle schemas/databases in the database server.
145 items:
146 properties:
147 oracleTables:
148 description: Tables in the database.
149 items:
150 properties:
151 oracleColumns:
152 description: Oracle columns in the schema. When
153 unspecified as part of include/exclude objects,
154 includes/excludes everything.
155 items:
156 properties:
157 column:
158 description: Column name.
159 type: string
160 dataType:
161 description: |-
162 The Oracle data type. Full data types list can be found here:
163 https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html.
164 type: string
165 encoding:
166 description: Column encoding.
167 type: string
168 length:
169 description: Column length.
170 type: integer
171 nullable:
172 description: Whether or not the column can
173 accept a null value.
174 type: boolean
175 ordinalPosition:
176 description: The ordinal position of the
177 column in the table.
178 type: integer
179 precision:
180 description: Column precision.
181 type: integer
182 primaryKey:
183 description: Whether or not the column represents
184 a primary key.
185 type: boolean
186 scale:
187 description: Column scale.
188 type: integer
189 type: object
190 type: array
191 table:
192 description: Table name.
193 type: string
194 required:
195 - table
196 type: object
197 type: array
198 schema:
199 description: Schema name.
200 type: string
201 required:
202 - schema
203 type: object
204 type: array
205 required:
206 - oracleSchemas
207 type: object
208 postgresqlExcludedObjects:
209 description: PostgreSQL data source objects to avoid backfilling.
210 properties:
211 postgresqlSchemas:
212 description: PostgreSQL schemas on the server.
213 items:
214 properties:
215 postgresqlTables:
216 description: Tables in the schema.
217 items:
218 properties:
219 postgresqlColumns:
220 description: PostgreSQL columns in the schema.
221 When unspecified as part of include/exclude
222 objects, includes/excludes everything.
223 items:
224 properties:
225 column:
226 description: Column name.
227 type: string
228 dataType:
229 description: |-
230 The PostgreSQL data type. Full data types list can be found here:
231 https://www.postgresql.org/docs/current/datatype.html.
232 type: string
233 length:
234 description: Column length.
235 type: integer
236 nullable:
237 description: Whether or not the column can
238 accept a null value.
239 type: boolean
240 ordinalPosition:
241 description: The ordinal position of the
242 column in the table.
243 type: integer
244 precision:
245 description: Column precision.
246 type: integer
247 primaryKey:
248 description: Whether or not the column represents
249 a primary key.
250 type: boolean
251 scale:
252 description: Column scale.
253 type: integer
254 type: object
255 type: array
256 table:
257 description: Table name.
258 type: string
259 required:
260 - table
261 type: object
262 type: array
263 schema:
264 description: Database name.
265 type: string
266 required:
267 - schema
268 type: object
269 type: array
270 required:
271 - postgresqlSchemas
272 type: object
273 type: object
274 backfillNone:
275 description: Backfill strategy to disable automatic backfill for the
276 Stream's objects.
277 type: object
278 x-kubernetes-preserve-unknown-fields: true
279 customerManagedEncryptionKey:
280 description: |-
281 Immutable. A reference to a KMS encryption key. If provided, it will be used to encrypt the data. If left blank, data
282 will be encrypted using an internal Stream-specific encryption key provisioned through KMS.
283 type: string
284 desiredState:
285 description: Desired state of the Stream. Set this field to 'RUNNING'
286 to start the stream, and 'PAUSED' to pause the stream.
287 type: string
288 destinationConfig:
289 description: Destination connection profile configuration.
290 properties:
291 bigqueryDestinationConfig:
292 description: A configuration for how data should be loaded to
293 Cloud Storage.
294 properties:
295 dataFreshness:
296 description: |-
297 The guaranteed data freshness (in seconds) when querying tables created by the stream.
298 Editing this field will only affect new tables created in the future, but existing tables
299 will not be impacted. Lower values mean that queries will return fresher data, but may result in higher cost.
300 A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s.
301 type: string
302 singleTargetDataset:
303 description: A single target dataset to which all data will
304 be streamed.
305 properties:
306 datasetId:
307 description: |-
308 Dataset ID in the format projects/{project}/datasets/{dataset_id} or
309 {project}:{dataset_id}.
310 type: string
311 required:
312 - datasetId
313 type: object
314 sourceHierarchyDatasets:
315 description: Destination datasets are created so that hierarchy
316 of the destination data objects matches the source hierarchy.
317 properties:
318 datasetTemplate:
319 description: Dataset template used for dynamic dataset
320 creation.
321 properties:
322 datasetIdPrefix:
323 description: |-
324 If supplied, every created dataset will have its name prefixed by the provided value.
325 The prefix and name will be separated by an underscore. i.e. _.
326 type: string
327 kmsKeyName:
328 description: |-
329 Immutable. Describes the Cloud KMS encryption key that will be used to protect destination BigQuery
330 table. The BigQuery Service Account associated with your project requires access to this
331 encryption key. i.e. projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{cryptoKey}.
332 See https://cloud.google.com/bigquery/docs/customer-managed-encryption for more information.
333 type: string
334 location:
335 description: |-
336 The geographic location where the dataset should reside.
337 See https://cloud.google.com/bigquery/docs/locations for supported locations.
338 type: string
339 required:
340 - location
341 type: object
342 required:
343 - datasetTemplate
344 type: object
345 type: object
346 destinationConnectionProfile:
347 description: 'Immutable. Destination connection profile resource.
348 Format: projects/{project}/locations/{location}/connectionProfiles/{name}.'
349 type: string
350 gcsDestinationConfig:
351 description: A configuration for how data should be loaded to
352 Cloud Storage.
353 properties:
354 avroFileFormat:
355 description: AVRO file format configuration.
356 type: object
357 x-kubernetes-preserve-unknown-fields: true
358 fileRotationInterval:
359 description: |-
360 The maximum duration for which new events are added before a file is closed and a new file is created.
361 A duration in seconds with up to nine fractional digits, terminated by 's'. Example: "3.5s". Defaults to 900s.
362 type: string
363 fileRotationMb:
364 description: The maximum file size to be saved in the bucket.
365 type: integer
366 jsonFileFormat:
367 description: JSON file format configuration.
368 properties:
369 compression:
370 description: 'Compression of the loaded JSON file. Possible
371 values: ["NO_COMPRESSION", "GZIP"].'
372 type: string
373 schemaFileFormat:
374 description: 'The schema file format along JSON data files.
375 Possible values: ["NO_SCHEMA_FILE", "AVRO_SCHEMA_FILE"].'
376 type: string
377 type: object
378 path:
379 description: Path inside the Cloud Storage bucket to write
380 data to.
381 type: string
382 type: object
383 required:
384 - destinationConnectionProfile
385 type: object
386 displayName:
387 description: Display name.
388 type: string
389 location:
390 description: Immutable. The name of the location this stream is located
391 in.
392 type: string
393 projectRef:
394 description: The project that this resource belongs to.
395 oneOf:
396 - not:
397 required:
398 - external
399 required:
400 - name
401 - not:
402 anyOf:
403 - required:
404 - name
405 - required:
406 - namespace
407 required:
408 - external
409 properties:
410 external:
411 description: 'Allowed value: The `name` field of a `Project` resource.'
412 type: string
413 name:
414 description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
415 type: string
416 namespace:
417 description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
418 type: string
419 type: object
420 resourceID:
421 description: Immutable. Optional. The streamId of the resource. Used
422 for creation and acquisition. When unset, the value of `metadata.name`
423 is used as the default.
424 type: string
425 sourceConfig:
426 description: Source connection profile configuration.
427 properties:
428 mysqlSourceConfig:
429 description: MySQL data source configuration.
430 properties:
431 excludeObjects:
432 description: MySQL objects to exclude from the stream.
433 properties:
434 mysqlDatabases:
435 description: MySQL databases on the server.
436 items:
437 properties:
438 database:
439 description: Database name.
440 type: string
441 mysqlTables:
442 description: Tables in the database.
443 items:
444 properties:
445 mysqlColumns:
446 description: MySQL columns in the schema.
447 When unspecified as part of include/exclude
448 objects, includes/excludes everything.
449 items:
450 properties:
451 collation:
452 description: Column collation.
453 type: string
454 column:
455 description: Column name.
456 type: string
457 dataType:
458 description: |-
459 The MySQL data type. Full data types list can be found here:
460 https://dev.mysql.com/doc/refman/8.0/en/data-types.html.
461 type: string
462 length:
463 description: Column length.
464 type: integer
465 nullable:
466 description: Whether or not the column
467 can accept a null value.
468 type: boolean
469 ordinalPosition:
470 description: The ordinal position of
471 the column in the table.
472 type: integer
473 primaryKey:
474 description: Whether or not the column
475 represents a primary key.
476 type: boolean
477 type: object
478 type: array
479 table:
480 description: Table name.
481 type: string
482 required:
483 - table
484 type: object
485 type: array
486 required:
487 - database
488 type: object
489 type: array
490 required:
491 - mysqlDatabases
492 type: object
493 includeObjects:
494 description: MySQL objects to retrieve from the source.
495 properties:
496 mysqlDatabases:
497 description: MySQL databases on the server.
498 items:
499 properties:
500 database:
501 description: Database name.
502 type: string
503 mysqlTables:
504 description: Tables in the database.
505 items:
506 properties:
507 mysqlColumns:
508 description: MySQL columns in the schema.
509 When unspecified as part of include/exclude
510 objects, includes/excludes everything.
511 items:
512 properties:
513 collation:
514 description: Column collation.
515 type: string
516 column:
517 description: Column name.
518 type: string
519 dataType:
520 description: |-
521 The MySQL data type. Full data types list can be found here:
522 https://dev.mysql.com/doc/refman/8.0/en/data-types.html.
523 type: string
524 length:
525 description: Column length.
526 type: integer
527 nullable:
528 description: Whether or not the column
529 can accept a null value.
530 type: boolean
531 ordinalPosition:
532 description: The ordinal position of
533 the column in the table.
534 type: integer
535 primaryKey:
536 description: Whether or not the column
537 represents a primary key.
538 type: boolean
539 type: object
540 type: array
541 table:
542 description: Table name.
543 type: string
544 required:
545 - table
546 type: object
547 type: array
548 required:
549 - database
550 type: object
551 type: array
552 required:
553 - mysqlDatabases
554 type: object
555 maxConcurrentBackfillTasks:
556 description: |-
557 Maximum number of concurrent backfill tasks. The number should be non negative.
558 If not set (or set to 0), the system's default value will be used.
559 type: integer
560 maxConcurrentCdcTasks:
561 description: |-
562 Maximum number of concurrent CDC tasks. The number should be non negative.
563 If not set (or set to 0), the system's default value will be used.
564 type: integer
565 type: object
566 oracleSourceConfig:
567 description: MySQL data source configuration.
568 properties:
569 dropLargeObjects:
570 description: Configuration to drop large object values.
571 type: object
572 x-kubernetes-preserve-unknown-fields: true
573 excludeObjects:
574 description: Oracle objects to exclude from the stream.
575 properties:
576 oracleSchemas:
577 description: Oracle schemas/databases in the database
578 server.
579 items:
580 properties:
581 oracleTables:
582 description: Tables in the database.
583 items:
584 properties:
585 oracleColumns:
586 description: Oracle columns in the schema.
587 When unspecified as part of include/exclude
588 objects, includes/excludes everything.
589 items:
590 properties:
591 column:
592 description: Column name.
593 type: string
594 dataType:
595 description: |-
596 The Oracle data type. Full data types list can be found here:
597 https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html.
598 type: string
599 encoding:
600 description: Column encoding.
601 type: string
602 length:
603 description: Column length.
604 type: integer
605 nullable:
606 description: Whether or not the column
607 can accept a null value.
608 type: boolean
609 ordinalPosition:
610 description: The ordinal position of
611 the column in the table.
612 type: integer
613 precision:
614 description: Column precision.
615 type: integer
616 primaryKey:
617 description: Whether or not the column
618 represents a primary key.
619 type: boolean
620 scale:
621 description: Column scale.
622 type: integer
623 type: object
624 type: array
625 table:
626 description: Table name.
627 type: string
628 required:
629 - table
630 type: object
631 type: array
632 schema:
633 description: Schema name.
634 type: string
635 required:
636 - schema
637 type: object
638 type: array
639 required:
640 - oracleSchemas
641 type: object
642 includeObjects:
643 description: Oracle objects to retrieve from the source.
644 properties:
645 oracleSchemas:
646 description: Oracle schemas/databases in the database
647 server.
648 items:
649 properties:
650 oracleTables:
651 description: Tables in the database.
652 items:
653 properties:
654 oracleColumns:
655 description: Oracle columns in the schema.
656 When unspecified as part of include/exclude
657 objects, includes/excludes everything.
658 items:
659 properties:
660 column:
661 description: Column name.
662 type: string
663 dataType:
664 description: |-
665 The Oracle data type. Full data types list can be found here:
666 https://docs.oracle.com/en/database/oracle/oracle-database/21/sqlrf/Data-Types.html.
667 type: string
668 encoding:
669 description: Column encoding.
670 type: string
671 length:
672 description: Column length.
673 type: integer
674 nullable:
675 description: Whether or not the column
676 can accept a null value.
677 type: boolean
678 ordinalPosition:
679 description: The ordinal position of
680 the column in the table.
681 type: integer
682 precision:
683 description: Column precision.
684 type: integer
685 primaryKey:
686 description: Whether or not the column
687 represents a primary key.
688 type: boolean
689 scale:
690 description: Column scale.
691 type: integer
692 type: object
693 type: array
694 table:
695 description: Table name.
696 type: string
697 required:
698 - table
699 type: object
700 type: array
701 schema:
702 description: Schema name.
703 type: string
704 required:
705 - schema
706 type: object
707 type: array
708 required:
709 - oracleSchemas
710 type: object
711 maxConcurrentBackfillTasks:
712 description: |-
713 Maximum number of concurrent backfill tasks. The number should be non negative.
714 If not set (or set to 0), the system's default value will be used.
715 type: integer
716 maxConcurrentCdcTasks:
717 description: |-
718 Maximum number of concurrent CDC tasks. The number should be non negative.
719 If not set (or set to 0), the system's default value will be used.
720 type: integer
721 streamLargeObjects:
722 description: Configuration to drop large object values.
723 type: object
724 x-kubernetes-preserve-unknown-fields: true
725 type: object
726 postgresqlSourceConfig:
727 description: PostgreSQL data source configuration.
728 properties:
729 excludeObjects:
730 description: PostgreSQL objects to exclude from the stream.
731 properties:
732 postgresqlSchemas:
733 description: PostgreSQL schemas on the server.
734 items:
735 properties:
736 postgresqlTables:
737 description: Tables in the schema.
738 items:
739 properties:
740 postgresqlColumns:
741 description: PostgreSQL columns in the schema.
742 When unspecified as part of include/exclude
743 objects, includes/excludes everything.
744 items:
745 properties:
746 column:
747 description: Column name.
748 type: string
749 dataType:
750 description: |-
751 The PostgreSQL data type. Full data types list can be found here:
752 https://www.postgresql.org/docs/current/datatype.html.
753 type: string
754 length:
755 description: Column length.
756 type: integer
757 nullable:
758 description: Whether or not the column
759 can accept a null value.
760 type: boolean
761 ordinalPosition:
762 description: The ordinal position of
763 the column in the table.
764 type: integer
765 precision:
766 description: Column precision.
767 type: integer
768 primaryKey:
769 description: Whether or not the column
770 represents a primary key.
771 type: boolean
772 scale:
773 description: Column scale.
774 type: integer
775 type: object
776 type: array
777 table:
778 description: Table name.
779 type: string
780 required:
781 - table
782 type: object
783 type: array
784 schema:
785 description: Database name.
786 type: string
787 required:
788 - schema
789 type: object
790 type: array
791 required:
792 - postgresqlSchemas
793 type: object
794 includeObjects:
795 description: PostgreSQL objects to retrieve from the source.
796 properties:
797 postgresqlSchemas:
798 description: PostgreSQL schemas on the server.
799 items:
800 properties:
801 postgresqlTables:
802 description: Tables in the schema.
803 items:
804 properties:
805 postgresqlColumns:
806 description: PostgreSQL columns in the schema.
807 When unspecified as part of include/exclude
808 objects, includes/excludes everything.
809 items:
810 properties:
811 column:
812 description: Column name.
813 type: string
814 dataType:
815 description: |-
816 The PostgreSQL data type. Full data types list can be found here:
817 https://www.postgresql.org/docs/current/datatype.html.
818 type: string
819 length:
820 description: Column length.
821 type: integer
822 nullable:
823 description: Whether or not the column
824 can accept a null value.
825 type: boolean
826 ordinalPosition:
827 description: The ordinal position of
828 the column in the table.
829 type: integer
830 precision:
831 description: Column precision.
832 type: integer
833 primaryKey:
834 description: Whether or not the column
835 represents a primary key.
836 type: boolean
837 scale:
838 description: Column scale.
839 type: integer
840 type: object
841 type: array
842 table:
843 description: Table name.
844 type: string
845 required:
846 - table
847 type: object
848 type: array
849 schema:
850 description: Database name.
851 type: string
852 required:
853 - schema
854 type: object
855 type: array
856 required:
857 - postgresqlSchemas
858 type: object
859 maxConcurrentBackfillTasks:
860 description: |-
861 Maximum number of concurrent backfill tasks. The number should be non
862 negative. If not set (or set to 0), the system's default value will be used.
863 type: integer
864 publication:
865 description: |-
866 The name of the publication that includes the set of all tables
867 that are defined in the stream's include_objects.
868 type: string
869 replicationSlot:
870 description: |-
871 The name of the logical replication slot that's configured with
872 the pgoutput plugin.
873 type: string
874 required:
875 - publication
876 - replicationSlot
877 type: object
878 sourceConnectionProfile:
879 description: 'Immutable. Source connection profile resource. Format:
880 projects/{project}/locations/{location}/connectionProfiles/{name}.'
881 type: string
882 required:
883 - sourceConnectionProfile
884 type: object
885 required:
886 - destinationConfig
887 - displayName
888 - location
889 - projectRef
890 - sourceConfig
891 type: object
892 status:
893 properties:
894 conditions:
895 description: Conditions represent the latest available observation
896 of the resource's current state.
897 items:
898 properties:
899 lastTransitionTime:
900 description: Last time the condition transitioned from one status
901 to another.
902 type: string
903 message:
904 description: Human-readable message indicating details about
905 last transition.
906 type: string
907 reason:
908 description: Unique, one-word, CamelCase reason for the condition's
909 last transition.
910 type: string
911 status:
912 description: Status is the status of the condition. Can be True,
913 False, Unknown.
914 type: string
915 type:
916 description: Type is the type of the condition.
917 type: string
918 type: object
919 type: array
920 name:
921 description: The stream's name.
922 type: string
923 observedGeneration:
924 description: ObservedGeneration is the generation of the resource
925 that was most recently observed by the Config Connector controller.
926 If this is equal to metadata.generation, then that means that the
927 current reported status reflects the most recent desired state of
928 the resource.
929 type: integer
930 state:
931 description: The state of the stream.
932 type: string
933 type: object
934 required:
935 - spec
936 type: object
937 served: true
938 storage: true
939 subresources:
940 status: {}
941status:
942 acceptedNames:
943 kind: ""
944 plural: ""
945 conditions: []
946 storedVersions: []
View as plain text