Sample Output for File-Based Connectors

If you have not used JSON before, take a look at Understanding JSON first.

For file-based connectors that use HTTP requests, the header looks like this:

User-Agent: DataSift-Push-Delivery-Service/1.0
Host: example.com  
Accept: \*/\*  
X-Datasift-Hash: c426ad575d435e5bc68a6edf125026c4  
X-Datasift-Hash-Type: stream  
X-Datasift-Id: 92186020677f1881aab7cddb28fa805c  
Expect: 100-continue

The output format depends on the setting you choose for output_params.format for the connector you use.

Metadata + array of interactions

To select this format, set your output_params.format to json_meta.

For all file-based connectors, the payload is a JSON object containing metadata plus an array containing the JSON objects from your stream.

{
    "id": "93186020677f1881aab7cddb28fa805c",
    "hash": "c426dd575d435e5bc68a6edf125026c4",
    "hash_type": "stream",
    "count": 2,
    "delivered_at": "Fri, 17 Aug 2012 14:23:00 +0000",
    "interactions": [
        {
            "interaction": {
                "source": "web",
                "author": {
                    "username": "johndoe",
                    "name": "John Doe",
                    "id": 10750902,
                    "avatar": "http://a0.twimg.com/profile_images/1111111111/example.jpeg",
                    "link": "http://twitter.com/johndoe"
                },
                "type": "twitter",
                "created_at": "Fri, 17 Aug 2012 14:13:08 +0000",
                "content": "I like ice cream!",
                "id": "1e1e875ab43fa233e074337458bc1dca",
                "link": "http://twitter.com/johndoe/statuses/111111111111111111",
                "geo": {
                    "latitude": 42.376104,
                    "longitude": -71.237189
                }
            },
            "twitter": {
                "created_at": "Fri, 17 Aug 2012 14:13:08 +0000",
                "domains": [
                    "4sq.com"
                ],
                "geo": {
                    "latitude": 42.376104,
                    "longitude": -71.237189
                },
                "id": "111111111111111111",
                "links": [
                    "http://4sq.com/NLM3gD"
                ],
                "mentions": [
                    "beyonce",
                    "ladygaga"
                ],
                "place": {
                    "id": "90ad0a08b3333d6d",
                    "url": "http://api.twitter.com/1/geo/id/example.json",
                    "place_type": "poi",
                    "country": "United States",
                    "country_code": "US",
                    "full_name": "Cafe On the Common, Waltham",
                    "name": "Cafe On the Common"
                },
                "source": "<a href=\"http://example.com\" rel=\"nofollow\">example</a>",
                "text": "I like ice cream",
                "user": {
                    "name": "John Doe",
                    "url": "http://about.me/John Doe",
                    "description": "all my tweets...",
                    "location": "London",
                    "statuses_count": 9689,
                    "followers_count": 2054,
                    "friends_count": 2016,
                    "screen_name": "johndoe",
                    "lang": "en",
                    "time_zone": "Eastern Time (US & Canada)",
                    "utc_offset": -18000,
                    "listed_count": 118,
                    "id": 11111111,
                    "id_str": "11111111",
                    "geo_enabled": true,
                    "created_at": "Fri, 30 Nov 2007 21:26:38 +0000"
                }
            }
        },
        {
            "demographic": {
                "gender": "mostly_male"
            },
            "interaction": {
                "source": "example",
                "author": {
                    "username": "JohnDoe",
                    "name": "John Doe",
                    "id": 11111111,
                    "avatar": "http://a0.twimg.com/profile_images/1111111111/example.jpg",
                    "link": "http://twitter.com/JohnDoe"
                },
                "type": "twitter",
                "created_at": "Fri, 17 Aug 2012 14:13:09 +0000",
                "content": "I love ice cream!",
                "id": "1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a",
                "link": "http://twitter.com/JohnDoe/statuses/111111111111111111",
                "geo": {
                    "latitude": 41.48454863,
                    "longitude": -72.79693173
                }
            },
            "twitter": {
                "created_at": "Fri, 17 Aug 2012 14:13:09 +0000",
                "domains": [
                    "4sq.com"
                ],
                "geo": {
                    "latitude": 41.48454863,
                    "longitude": -72.79693173
                },
                "id": "111111111111111111",
                "links": [
                    "http://4sq.com/NMH47L"
                ],
                "place": {
                    "id": "e5ac52573b5f3333",
                    "url": "http://api.twitter.com/1/geo/id/1a1a1a1a1a1a1a1a.json",
                    "place_type": "poi",
                    "country": "United States",
                    "country_code": "US",
                    "full_name": "Willow Farm",
                    "name": "Willow Farm"
                },
                "source": "<a href=\"http://example.com\" rel=\"nofollow\">example</a>",
                "text": "I love ice cream",
                "user": {
                    "name": "John Doe",
                    "description": "Man of mystery\n",
                    "location": "Main street",
                    "statuses_count": 10073,
                    "followers_count": 444,
                    "friends_count": 533,
                    "screen_name": "John Doe",
                    "lang": "en",
                    "time_zone": "Quito",
                    "utc_offset": -18000,
                    "listed_count": 9,
                    "id": 11111111,
                    "id_str": "11111111",
                    "geo_enabled": true,
                    "created_at": "Sun, 26 Apr 2009 23:38:45 +0000"
                }
            }
        }
    ]
}

Array of interactions only

To select this format, set your output_params.format to json_array.

For all file-based connectors, the payload is a JSON object containing metadata plus an array containing the JSON objects from your stream.

[
    {
        "interaction": {
            "source": "web",
            "author": {
                "username": "johndoe",
                "name": "John Doe",
                "id": 10750902,
                "avatar": "http://a0.twimg.com/profile_images/1111111111/example.jpeg",
                "link": "http://twitter.com/johndoe"
            },
            "type": "twitter",
            "created_at": "Fri, 17 Aug 2012 14:13:08 +0000",
            "content": "I like ice cream!",
            "id": "1e1e875ab43fa233e074337458bc1dca",
            "link": "http://twitter.com/johndoe/statuses/111111111111111111",
            "geo": {
                "latitude": 42.376104,
                "longitude": -71.237189
            }
        },
        "twitter": {
            "created_at": "Fri, 17 Aug 2012 14:13:08 +0000",
            "domains": [
                "4sq.com"
            ],
            "geo": {
                "latitude": 42.376104,
                "longitude": -71.237189
            },
            "id": "111111111111111111",
            "links": [
                "http://4sq.com/NLM3gD"
            ],
            "mentions": [
                "beyonce",
                "ladygaga"
            ],
            "place": {
                "id": "90ad0a08b3333d6d",
                "url": "http://api.twitter.com/1/geo/id/example.json",
                "place_type": "poi",
                "country": "United States",
                "country_code": "US",
                "full_name": "Cafe On the Common, Waltham",
                "name": "Cafe On the Common"
            },
            "source": "<a href=\"http://example.com\" rel=\"nofollow\">example</a>",
            "text": "I like ice cream",
            "user": {
                "name": "John Doe",
                "url": "http://about.me/John Doe",
                "description": "all my tweets...",
                "location": "London",
                "statuses_count": 9689,
                "followers_count": 2054,
                "friends_count": 2016,
                "screen_name": "johndoe",
                "lang": "en",
                "time_zone": "Eastern Time (US & Canada)",
                "utc_offset": -18000,
                "listed_count": 118,
                "id": 11111111,
                "id_str": "11111111",
                "geo_enabled": true,
                "created_at": "Fri, 30 Nov 2007 21:26:38 +0000"
            }
        }
    },
    {
        "demographic": {
            "gender": "mostly_male"
        },
        "interaction": {
            "source": "web",
            "author": {
                "username": "JohnDoe",
                "name": "John Doe",
                "id": 11111111,
                "avatar": "http://a0.twimg.com/profile_images/1111111111/example.jpg",
                "link": "http://twitter.com/JohnDoe"
            },
            "type": "twitter",
            "created_at": "Fri, 17 Aug 2012 14:13:09 +0000",
            "content": "I love ice cream!",
            "id": "1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a",
            "link": "http://twitter.com/JohnDoe/statuses/111111111111111111",
            "geo": {
                "latitude": 41.48454863,
                "longitude": -72.79693173
            }
        },
        "twitter": {
            "created_at": "Fri, 17 Aug 2012 14:13:09 +0000",
            "domains": [
                "4sq.com"
            ],
            "geo": {
                "latitude": 41.48454863,
                "longitude": -72.79693173
            },
            "id": "111111111111111111",
            "links": [
                "http://4sq.com/NMH47L"
            ],
            "place": {
                "id": "e5ac52573b5f3333",
                "url": "http://api.twitter.com/1/geo/id/1a1a1a1a1a1a1a1a.json",
                "place_type": "poi",
                "country": "United States",
                "country_code": "US",
                "full_name": "Willow Farm",
                "name": "Willow Farm"
            },
            "source": "<a href=\"http://example.com\" rel=\"nofollow\">example</a>",
            "text": "I love ice cream",
            "user": {
                "name": "John Doe",
                "description": "Man of mystery\n",
                "location": "Main street",
                "statuses_count": 10073,
                "followers_count": 444,
                "friends_count": 533,
                "screen_name": "John Doe",
                "lang": "en",
                "time_zone": "Quito",
                "utc_offset": -18000,
                "listed_count": 9,
                "id": 11111111,
                "id_str": "11111111",
                "geo_enabled": true,
                "created_at": "Sun, 26 Apr 2009 23:38:45 +0000"
            }
        }
    }
]

Individual interactions only

To select this format, set your output_params.format to json_new_line.

For all file-based connectors, the payload is a JSON object containing metadata plus an array containing the JSON objects from your stream.

{
    "interaction": {
        "source": "web",
        "author": {
            "username": "johndoe",
            "name": "John Doe",
            "id": 10750902,
            "avatar": "http://a0.twimg.com/profile_images/1111111111/example.jpeg",
            "link": "http://twitter.com/johndoe"
        },
        "type": "twitter",
        "created_at": "Fri, 17 Aug 2012 14:13:08 +0000",
        "content": "I like ice cream!",
        "id": "1e1e875ab43fa233e074337458bc1dca",
        "link": "http://twitter.com/johndoe/statuses/111111111111111111",
        "geo": {
            "latitude": 42.376104,
            "longitude": -71.237189
        }
    },
    "twitter": {
        "created_at": "Fri, 17 Aug 2012 14:13:08 +0000",
        "domains": [
            "4sq.com"
        ],
        "geo": {
            "latitude": 42.376104,
            "longitude": -71.237189
        },
        "id": "111111111111111111",
        "links": [
            "http://4sq.com/NLM3gD"
        ],
        "mentions": [
            "beyonce",
            "ladygaga"
        ],
        "place": {
            "id": "90ad0a08b3333d6d",
            "url": "http://api.twitter.com/1/geo/id/example.json",
            "place_type": "poi",
            "country": "United States",
            "country_code": "US",
            "full_name": "Cafe On the Common, Waltham",
            "name": "Cafe On the Common"
        },
        "source": "<a href=\"http://example.com\" rel=\"nofollow\">example</a>",
        "text": "I like ice cream",
        "user": {
            "name": "John Doe",
            "url": "http://about.me/John Doe",
            "description": "all my tweets...",
            "location": "London",
            "statuses_count": 9689,
            "followers_count": 2054,
            "friends_count": 2016,
            "screen_name": "johndoe",
            "lang": "en",
            "time_zone": "Eastern Time (US & Canada)",
            "utc_offset": -18000,
            "listed_count": 118,
            "id": 11111111,
            "id_str": "11111111",
            "geo_enabled": true,
            "created_at": "Fri, 30 Nov 2007 21:26:38 +0000"
        }
    }
}

{
    "demographic": {
        "gender": "mostly_male"
    },
    "interaction": {
        "source": "web",
        "author": {
            "username": "JohnDoe",
            "name": "John Doe",
            "id": 11111111,
            "avatar": "http://a0.twimg.com/profile_images/1111111111/example.jpg",
            "link": "http://twitter.com/JohnDoe"
        },
        "type": "twitter",
        "created_at": "Fri, 17 Aug 2012 14:13:09 +0000",
        "content": "I love ice cream!",
        "id": "1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a",
        "link": "http://twitter.com/JohnDoe/statuses/111111111111111111",
        "geo": {
            "latitude": 41.48454863,
            "longitude": -72.79693173
        }
    },
    "twitter": {
        "created_at": "Fri, 17 Aug 2012 14:13:09 +0000",
        "domains": [
            "4sq.com"
        ],
        "geo": {
            "latitude": 41.48454863,
            "longitude": -72.79693173
        },
        "id": "111111111111111111",
        "links": [
            "http://4sq.com/NMH47L"
        ],
        "place": {
            "id": "e5ac52573b5f3333",
            "url": "http://api.twitter.com/1/geo/id/1a1a1a1a1a1a1a1a.json",
            "place_type": "poi",
            "country": "United States",
            "country_code": "US",
            "full_name": "Willow Farm",
            "name": "Willow Farm"
        },
        "source": "<a href=\"http://example.com\" rel=\"nofollow\">example</a>",
        "text": "I love ice cream",
        "user": {
            "name": "John Doe",
            "description": "Man of mystery\n",
            "location": "Main street",
            "statuses_count": 10073,
            "followers_count": 444,
            "friends_count": 533,
            "screen_name": "John Doe",
            "lang": "en",
            "time_zone": "Quito",
            "utc_offset": -18000,
            "listed_count": 9,
            "id": 11111111,
            "id_str": "11111111",
            "geo_enabled": true,
            "created_at": "Sun, 26 Apr 2009 23:38:45 +0000"
        }
    }
}

Basic interactions (CouchDB, ElasticSearch, MongoDB)

Each payload contains only basic interaction JSON document.

{
    "interactionId": "f35ace79b903eedfe5198f386d6fda0c",
    "subscriptionId": "ABC123456a891e3406789123216789",
    "hash": null,
    "hashType": null,
    "interaction": {
        "schema": {
            "version": 3
        },
        "source": "Twitter for Android",
        "type": "twitter",
        "created_at": "Thu, 09 May 2013 08:59:46 +0000",
        "content": "RT @Real_Liam_Payne: Thank you denmarkkkk :) love youuuu :)",
        "id": "f35ace79b903eedf75198f386d6fd40b",
        "author": {
            "hash_id": "c6add0a675830a785598a513d586203c"
        },
        "tags": [
            "type.share",
            "demo.tag",
            "another.one",
            "foobar"
        ]
    }
}

Basic interactions with Augmentations (CouchDB, ElasticSearch, MongoDB)

The payload is a full interaction with augmentations.

{
    "interactionId": "f35ace79b903eedfe5198f386d6fda0c",
    "subscriptionId": "ABC123456a891e3406789123216789",
    "hash": null,
    "hashType": null,
    "interaction": {
        "demographic": {
            "first_language": "English",
            "main_street": {
                "eat_and_drink_at": [
                    "Starbucks"
                ],
                "dressed_by": [
                    "Burberry"
                ]
            },
            "sex": "Female",
            "twitter": {
                "activity": "1-5 tweets/day",
                "accounts.large": [
                    "Mayo Clinic",
                    "Debby Ryan",
                    "Creative Commons",
                    "Rebecca Black",
                    "Nick Jonas",
                    "Jessie J",
                    "Wynn Resorts",
                    "Rihanna",
                    "Politico",
                    "Ariana Grande",
                    "Cody Simpson",
                    "Lord_Voldemort7",
                    "Jordin Sparks",
                    "Selena Gomez",
                    "Demi Lovato",
                    "Ashley Tisdale",
                    "Emily Osment",
                    "Psy"
                ]
            },
            "type": "People"
        },
        "interaction": {
            "schema": {
                "version": 3
            },
            "source": "Twitter for Android",
            "type": "twitter",
            "created_at": "Thu, 09 May 2013 08:59:46 +0000",
            "content": "RT @Real_Liam_Payne: Thank you denmarkkkk :) love youuuu :)",
            "id": "f35ace79b903eedf75198f386d6fd40b",
            "author": {
                "hash_id": "c6add0a675830a785598a513d586203c"
            },
            "tags": [
                "type.share",
                "demo.tag",
                "another.one",
                "foobar"
            ]
        },
        "klout": {
            "amplification": 2,
            "network": 1.47,
            "score": 41,
            "true_reach": 10
        },
        "language": {
            "tag": "en",
            "confidence": 63
        },
        "salience": {
            "content": {
                "sentiment": 4
            }
        },
        "twitter": {
            "id": "231419612755326460",
            "retweet": {
                "text": "Thank you denmarkkkk :) love youuuu :)",
                "id": "231419612755326460",
                "user": {
                    "description": "♥One Direction ♥ 1D is my life ♥ DIRECTIONER ♥' 0/5 D':♥ BFF Annissa ♥ Single ♥ Life is a wonderfull thing, and only wonderfull people can get it  ♥",
                    "statuses_count": 4711,
                    "followers_count": 329,
                    "friends_count": 1257,
                    "profile_image_url": "http://a0.twimg.com/profile_images/3632046312/c0d75d140a02775761b8d9ecffe13a7b_normal.jpeg",
                    "lang": "nl",
                    "time_zone": "Amsterdam",
                    "utc_offset": 3600,
                    "geo_enabled": true,
                    "favourites_count": 123,
                    "created_at": "Sat, 15 Oct 2011 08:04:25 +0000",
                    "hash_id": "c6cdd4a675830a795598e513d586203b"
                },
                "source": "Twitter for Android",
                "count": 32166,
                "created_at": "Thu, 09 May 2013 08:59:46 +0000",
                "lang": "en"
            },
            "retweeted": {
                "id": "312189094049411043",
                "user": {
                    "description": "its on like donkey kong",
                    "location": "UK",
                    "statuses_count": 7254,
                    "followers_count": 10365944,
                    "friends_count": 13981,
                    "profile_image_url": "http://a0.twimg.com/profile_images/a566362915/de1b239ca19b01ef71994c57ef46e92e_normal.jpeg",
                    "lang": "en",
                    "listed_count": 57226,
                    "geo_enabled": true,
                    "verified": true,
                    "favourites_count": 29,
                    "created_at": "Tue, 22 Jun 2010 10:19:08 +0000",
                    "hash_id": "e74afc43e08a7b3969846ea4b1b8c83f"
                },
                "source": "Twitter for iPhone",
                "created_at": "Sun, 05 May 2013 23:30:08 +0000"
            }
        }
    }
}

Basic interactions with Augmentations, MongoDB Date Formet (MongoDB)

Each payload is a full interaction with augmentations and a MongoDB-specific date representation.

{
    "interactionId": "f35ace79b903eedfe5198f386d6fda0c",
    "subscriptionId": "ABC123456a891e3406789123216789",
    "hash": null,
    "hashType": null,
    "interaction": {
        "demographic": {
            "first_language": "English",
            "main_street": {
                "eat_and_drink_at": [
                    "Starbucks"
                ],
                "dressed_by": [
                    "Burberry"
                ]
            },
            "sex": "Female",
            "twitter": {
                "activity": "1-5 tweets/day",
                "accounts.large": [
                    "Mayo Clinic",
                    "Debby Ryan",
                    "Creative Commons",
                    "Rebecca Black",
                    "Nick Jonas",
                    "Jessie J",
                    "Wynn Resorts",
                    "Rihanna",
                    "Politico",
                    "Ariana Grande",
                    "Cody Simpson",
                    "Lord_Voldemort7",
                    "Jordin Sparks",
                    "Selena Gomez",
                    "Demi Lovato",
                    "Ashley Tisdale",
                    "Emily Osment",
                    "Psy"
                ]
            },
            "type": "People"
        },
        "interaction": {
            "schema": {
                "version": 3
            },
            "source": "Twitter for Android",
            "type": "twitter",
            "created_at": "Thu, 09 May 2013 08:59:46 +0000",
            "content": "RT @Real_Liam_Payne: Thank you denmarkkkk :) love youuuu :)",
            "id": "f35ace79b903eedf85198f386d6fd40b",
            "author": {
                "hash_id": "c6add0a675830a785598a513d586203c"
            },
            "tags": [
                "type.share",
                "demo.tag",
                "another.one",
                "foobar"
            ]
        },
        "klout": {
            "amplification": 2,
            "network": 1.47,
            "score": 41,
            "true_reach": 10
        },
        "language": {
            "tag": "en",
            "confidence": 63
        },
        "salience": {
            "content": {
                "sentiment": 4
            }
        },
        "twitter": {
            "id": "231419612755326460",
            "retweet": {
                "text": "Thank you denmarkkkk :) love youuuu :)",
                "id": "231419612755326460",
                "user": {
                    "description": "♥One Direction ♥ 1D is my life ♥ DIRECTIONER ♥' 0/5 D':♥ BFF Annissa ♥ Single ♥ Life is a wonderfull thing, and only wonderfull people can get it  ♥",
                    "statuses_count": 4711,
                    "followers_count": 329,
                    "friends_count": 1257,
                    "profile_image_url": "http://a0.twimg.com/profile_images/3632046312/c0d75d140a02775761b8d9ecffe13a7b_normal.jpeg",
                    "lang": "nl",
                    "time_zone": "Amsterdam",
                    "utc_offset": 3600,
                    "geo_enabled": true,
                    "favourites_count": 123,
                    "created_at": "Sat, 15 Oct 2011 08:04:25 +0000",
                    "hash_id": "c6cdd4a675830a795598e513d586203b"
                },
                "source": "Twitter for Android",
                "count": 32166,
                "created_at": "Thu, 09 May 2013 08:59:46 +0000",
                "lang": "en"
            },
            "retweeted": {
                "id": "312189094049411043",
                "user": {
                    "description": "its on like donkey kong",
                    "location": "FR",
                    "statuses_count": 7254,
                    "followers_count": 10365944,
                    "friends_count": 13981,
                    "profile_image_url": "http://a0.twimg.com/profile_images/a566362915/de1b239ca19b01ef71994c57ef46e92e_normal.jpeg",
                    "lang": "en",
                    "listed_count": 57226,
                    "geo_enabled": false,
                    "verified": true,
                    "favourites_count": 29,
                    "created_at": "Tue, 22 Jun 2010 10:19:08 +0000",
                    "hash_id": "e74afc43e08a7b3969846ea4b1b8c83f"
                },
                "source": "Twitter for iPhone",
                "created_at": "Sun, 05 May 2013 23:30:08 +0000"
            }
        }
    },
    "created_at": ISODate("2013-05-09T08:59:46Z")
}