diff --git a/packages/histoire/src/node/commands/dev.ts b/packages/histoire/src/node/commands/dev.ts
index 3bd69736..4b4044ea 100644
--- a/packages/histoire/src/node/commands/dev.ts
+++ b/packages/histoire/src/node/commands/dev.ts
@@ -15,7 +15,7 @@ export async function devCommand (options: DevOptions) {
     const ctx = await createContext({
       mode: 'dev',
     })
-    const { server, close } = await createServer(ctx, options.port ?? 6006)
+    const { server, close } = await createServer(ctx, { port: options.port })
     server.printUrls()
 
     // Histoire config watcher
diff --git a/packages/histoire/src/node/server.ts b/packages/histoire/src/node/server.ts
index a67bbb14..c17bdfd5 100644
--- a/packages/histoire/src/node/server.ts
+++ b/packages/histoire/src/node/server.ts
@@ -9,7 +9,11 @@ import { useModuleLoader } from './load.js'
 import { wrapLogError } from './util/log.js'
 import { createMarkdownFilesWatcher, onMarkdownListChange } from './markdown.js'
 
-export async function createServer (ctx: Context, port: number) {
+export interface CreateServerOptions {
+  port?: number
+}
+
+export async function createServer (ctx: Context, options: CreateServerOptions = {}) {
   const server = await createViteServer(await getViteConfigWithPlugins(false, ctx))
   await server.pluginContainer.buildStart({})
 
@@ -36,7 +40,7 @@ export async function createServer (ctx: Context, port: number) {
   }
 
   // Wait for pre-bundling (in `listen()`)
-  await server.listen(port)
+  await server.listen(options.port ?? server.config.server?.port)
 
   const {
     clearCache,
diff --git a/packages/histoire/src/node/vite.ts b/packages/histoire/src/node/vite.ts
index d4cefce5..b448a497 100644
--- a/packages/histoire/src/node/vite.ts
+++ b/packages/histoire/src/node/vite.ts
@@ -105,9 +105,10 @@ async function mergeHistoireViteConfig (viteConfig: InlineConfig, ctx: Context)
 export async function getViteConfigWithPlugins (isServer: boolean, ctx: Context): Promise<InlineConfig> {
   const resolvedViteConfig = await resolveViteConfig(ctx)
 
-  const userViteConfig = await loadViteConfigFromFile({ command: ctx.mode === 'dev' ? 'serve' : 'build', mode: ctx.mode })
+  const userViteConfigFile = await loadViteConfigFromFile({ command: ctx.mode === 'dev' ? 'serve' : 'build', mode: ctx.mode })
+  const userViteConfig = mergeViteConfig(userViteConfigFile?.config ?? {}, { server: { port: 6006 } })
 
-  const inlineConfig = await mergeHistoireViteConfig(userViteConfig?.config ?? {}, ctx)
+  const inlineConfig = await mergeHistoireViteConfig(userViteConfig, ctx)
   const plugins: VitePlugin[] = []
 
   const hasPnpm = !!(await findUp(ctx.root, ['pnpm-lock.yaml']))