There is some generic sample code for generating a hash on this page:

https://wiki.openssl.org/index.php/EVP_Message_Digests

That particular code generates a SHA256 hash. To convert it to use SHA3 instead, replace the two instances of EVP_sha256() with one of EVP_sha3_224(), EVP_sha3_256(), EVP_sha3_384() or EVP_sha3_512() as appropriate.

On Ubuntu 18.04, with libssl-dev installed:

//main.cpp

#include <iostream>
#include <vector>
#include <sstream> //for std::ostringstream
#include <iomanip> //for std::setw, std::hex, and std::setfill
#include <openssl/evp.h> //for all other OpenSSL function calls
#include <openssl/sha.h> //for SHA512_DIGEST_LENGTH

//helper function to print the digest bytes as a hex string
std::string bytes_to_hex_string(const std::vector<uint8_t>& bytes)
{
    std::ostringstream stream;
    for (uint8_t b : bytes)
    {
        stream << std::setw(2) << std::setfill('0') << std::hex << static_cast<int>(b);
    }
    return stream.str();
}

//perform the SHA3-512 hash
std::string sha3_512(const std::string& input)
{
    uint32_t digest_length = SHA512_DIGEST_LENGTH;
    const EVP_MD* algorithm = EVP_sha3_512();
    uint8_t* digest = static_cast<uint8_t*>(OPENSSL_malloc(digest_length));
    EVP_MD_CTX* context = EVP_MD_CTX_new();
    EVP_DigestInit_ex(context, algorithm, nullptr);
    EVP_DigestUpdate(context, input.c_str(), input.size());
    EVP_DigestFinal_ex(context, digest, &digest_length);
    EVP_MD_CTX_destroy(context);
    std::string output = bytes_to_hex_string(std::vector<uint8_t>(digest, digest + digest_length));
    OPENSSL_free(digest);
    EVP_MD_CTX_free(context);
    return output;
}

int main()
{
    std::string output = sha3_512(std::string("abcdef"));
    std::cout << output << "\n";
    return 0;
}

Link with -lssl and -lcrypto to compile this code:

/usr/bin/g++-10 main.cpp -lssl -lcrypto

Output:

01309a45c57cd7faef9ee6bb95fed29e5e2e0312af12a95fffeee340e5e5948b4652d26ae4b75976a53cc1612141af6e24df36517a61f46a1a05f59cf667046a

Verify the output with this tool: https://emn178.github.io/online-tools/sha3_512.html

I have written a simple example which uses openssl SHA-3 implementation to calculate the digest of all the data read from stdin. With a command line parameter to specify SHA-3 digest length. Tested on Linux Parrot (distro based on Debian), libssl-dev 1.1.1c-1.

Link to the repository.

Or here is the source code:

#include <openssl/evp.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
/*
 * building on Debian requires libssl-dev package (sudo apt install libssl-dev)
 */

#define BUF_SIZE 1024

#define HANDLE_ERROR(msg) { fprintf(stderr, "%s\n", msg); exit(EXIT_FAILURE); }
#define HANDLE_ERROR2(msg, mdctx) { fprintf(stderr, "%s\n", msg); EVP_MD_CTX_destroy(mdctx); exit(EXIT_FAILURE); }

int main(int argc,  char * const argv[]) {

    int opt;
    char * endptr;
    char buffer[BUF_SIZE];
    int bytes_read;

    EVP_MD_CTX * mdctx;
    int val;
    unsigned char * digest;
    unsigned int digest_len;
    EVP_MD * algo = NULL;

    while ((opt = getopt(argc, argv, "t:")) != -1) {
        switch (opt) {
        case 't':
            errno = 0;

            val = strtol(optarg, &endptr, 10);

            if ((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN))
                    || (errno != 0 && val == 0)) {
                perror("Wrong value for t parameter");
                exit(EXIT_FAILURE);
            }

            if (endptr == optarg) {
                fprintf(stderr, "No value was found for t parameter\n");
                exit(EXIT_FAILURE);
            }

            switch (val) {
            case 224:
                algo = EVP_sha3_224();
                break;
            case 256:
                algo = EVP_sha3_256();
                break;
            case 384:
                algo = EVP_sha3_384();
                break;
            case 512:
                algo = EVP_sha3_512();
                break;
            default:
                fprintf(stderr,"Wrong value for t parameter (valid values: 224, 256, 384, 512)");
                exit(EXIT_FAILURE);
            }

            break;
        default: /* '?' */
            fprintf(stderr, "Usage: %s [-t sha3_size]\n"
                    "Example program which calculates SHA-3 hash of data read from stdin.\n"
                    "Uses openssl implementation of SHA-3 algorithm.\n"
                    "sha3_size can be: 224, 256, 384, 512. Default is 256.\n",
                    argv[0]);
            exit(EXIT_FAILURE);
        }
    }

    if (algo == NULL) {
        algo = EVP_sha3_256();
    }


    if ((mdctx = EVP_MD_CTX_create()) == NULL) {
        HANDLE_ERROR("EVP_MD_CTX_create() error")
    }

    // initialize digest engine
    if (EVP_DigestInit_ex(mdctx, algo, NULL) != 1) { // returns 1 if successful
        HANDLE_ERROR2("EVP_DigestInit_ex() error", mdctx)
    }

    while ((bytes_read = read(STDIN_FILENO, buffer, BUF_SIZE)) > 0) { // read returns 0 on EOF, -1 on error
        // provide data to digest engine
        if (EVP_DigestUpdate(mdctx, buffer, bytes_read) != 1) { // returns 1 if successful
            HANDLE_ERROR2("EVP_DigestUpdate() error", mdctx)
        }

    }

    if (bytes_read == -1) {
        perror("read error");
        exit(1);
    }

    digest_len = EVP_MD_size(algo);

    if ((digest = (unsigned char *)OPENSSL_malloc(digest_len)) == NULL) {
        HANDLE_ERROR2("OPENSSL_malloc() error", mdctx)
    }

    // produce digest
    if (EVP_DigestFinal_ex(mdctx, digest, &digest_len) != 1) { // returns 1 if successful
        OPENSSL_free(digest);
        HANDLE_ERROR2("EVP_DigestFinal_ex() error", mdctx)
    }

    for (int i = 0; i < digest_len; i++) {
        printf("%02x", digest[i]);
    }

    OPENSSL_free(digest);
    EVP_MD_CTX_destroy(mdctx);

    return 0;
}

after installing openssl and configuring PATH, include <openssl/evp.h> in your source file. here is description and example.